2024-11-14 07:15:44,338 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 07:15:44,352 main DEBUG Took 0.012114 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 07:15:44,352 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 07:15:44,353 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 07:15:44,353 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 07:15:44,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,361 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 07:15:44,372 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,374 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,374 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,375 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,377 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,378 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,379 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,379 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,380 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,381 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,382 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,382 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,383 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:15:44,384 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,384 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 07:15:44,385 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:15:44,387 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 07:15:44,389 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 07:15:44,390 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 07:15:44,391 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 07:15:44,392 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 07:15:44,400 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 07:15:44,402 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 07:15:44,404 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 07:15:44,405 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 07:15:44,405 main DEBUG createAppenders(={Console}) 2024-11-14 07:15:44,406 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 07:15:44,406 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 07:15:44,407 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 07:15:44,407 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 07:15:44,407 main DEBUG OutputStream closed 2024-11-14 07:15:44,408 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 07:15:44,408 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 07:15:44,408 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 07:15:44,484 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 07:15:44,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 07:15:44,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 07:15:44,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 07:15:44,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 07:15:44,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 07:15:44,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 07:15:44,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 07:15:44,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 07:15:44,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 07:15:44,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 07:15:44,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 07:15:44,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 07:15:44,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 07:15:44,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 07:15:44,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 07:15:44,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 07:15:44,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 07:15:44,502 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 07:15:44,502 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 07:15:44,503 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 07:15:44,504 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T07:15:44,742 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e 2024-11-14 07:15:44,745 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 07:15:44,745 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T07:15:44,753 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-14T07:15:44,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T07:15:44,792 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4, deleteOnExit=true 2024-11-14T07:15:44,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T07:15:44,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/test.cache.data in system properties and HBase conf 2024-11-14T07:15:44,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T07:15:44,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/hadoop.log.dir in system properties and HBase conf 2024-11-14T07:15:44,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T07:15:44,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T07:15:44,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T07:15:44,926 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T07:15:45,011 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T07:15:45,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T07:15:45,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T07:15:45,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T07:15:45,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T07:15:45,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T07:15:45,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T07:15:45,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T07:15:45,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T07:15:45,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T07:15:45,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/nfs.dump.dir in system properties and HBase conf 2024-11-14T07:15:45,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/java.io.tmpdir in system properties and HBase conf 2024-11-14T07:15:45,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T07:15:45,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T07:15:45,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T07:15:46,205 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T07:15:46,274 INFO [Time-limited test {}] log.Log(170): Logging initialized @2772ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T07:15:46,340 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T07:15:46,393 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T07:15:46,413 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T07:15:46,413 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T07:15:46,414 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T07:15:46,426 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T07:15:46,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/hadoop.log.dir/,AVAILABLE} 2024-11-14T07:15:46,430 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T07:15:46,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/java.io.tmpdir/jetty-localhost-36043-hadoop-hdfs-3_4_1-tests_jar-_-any-4277441217451944892/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T07:15:46,612 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:36043} 2024-11-14T07:15:46,612 INFO [Time-limited test {}] server.Server(415): Started @3111ms 2024-11-14T07:15:47,128 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T07:15:47,135 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T07:15:47,137 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T07:15:47,137 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T07:15:47,137 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T07:15:47,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/hadoop.log.dir/,AVAILABLE} 2024-11-14T07:15:47,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T07:15:47,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/java.io.tmpdir/jetty-localhost-46129-hadoop-hdfs-3_4_1-tests_jar-_-any-6246196217637362089/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T07:15:47,237 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:46129} 2024-11-14T07:15:47,237 INFO [Time-limited test {}] server.Server(415): Started @3737ms 2024-11-14T07:15:47,285 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T07:15:48,339 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/data/data2/current/BP-1354491081-172.17.0.2-1731568545633/current, will proceed with Du for space computation calculation, 2024-11-14T07:15:48,339 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/data/data1/current/BP-1354491081-172.17.0.2-1731568545633/current, will proceed with Du for space computation calculation, 2024-11-14T07:15:48,371 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T07:15:48,420 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20b5ee2bc7e853ed with lease ID 0xab80491fd2ae6396: Processing first storage report for DS-bbe5f45d-0a0e-4dc5-ac16-fa0b5dce602f from datanode DatanodeRegistration(127.0.0.1:33971, datanodeUuid=c2719293-c0db-4b29-af53-f82a81ce677a, infoPort=37215, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1457088147;c=1731568545633) 2024-11-14T07:15:48,421 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20b5ee2bc7e853ed with lease ID 0xab80491fd2ae6396: from storage DS-bbe5f45d-0a0e-4dc5-ac16-fa0b5dce602f node DatanodeRegistration(127.0.0.1:33971, datanodeUuid=c2719293-c0db-4b29-af53-f82a81ce677a, infoPort=37215, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1457088147;c=1731568545633), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T07:15:48,422 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20b5ee2bc7e853ed with lease ID 0xab80491fd2ae6396: Processing first storage report for DS-24837263-6861-4f38-9be0-fe8a59964717 from datanode DatanodeRegistration(127.0.0.1:33971, datanodeUuid=c2719293-c0db-4b29-af53-f82a81ce677a, infoPort=37215, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1457088147;c=1731568545633) 2024-11-14T07:15:48,422 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20b5ee2bc7e853ed with lease ID 0xab80491fd2ae6396: from storage DS-24837263-6861-4f38-9be0-fe8a59964717 node DatanodeRegistration(127.0.0.1:33971, datanodeUuid=c2719293-c0db-4b29-af53-f82a81ce677a, infoPort=37215, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1457088147;c=1731568545633), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T07:15:48,500 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e 2024-11-14T07:15:48,562 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/zookeeper_0, clientPort=58901, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T07:15:48,571 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58901 2024-11-14T07:15:48,580 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:15:48,583 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:15:48,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741825_1001 (size=7) 2024-11-14T07:15:49,196 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 with version=8 2024-11-14T07:15:49,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/hbase-staging 2024-11-14T07:15:49,287 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T07:15:49,603 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c7aa6e2790b3:0 server-side Connection retries=45 2024-11-14T07:15:49,611 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:15:49,611 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T07:15:49,617 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T07:15:49,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:15:49,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T07:15:49,757 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T07:15:49,804 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T07:15:49,812 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T07:15:49,815 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T07:15:49,836 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8896 (auto-detected) 2024-11-14T07:15:49,837 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T07:15:49,853 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37073 2024-11-14T07:15:49,873 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37073 connecting to ZooKeeper ensemble=127.0.0.1:58901 2024-11-14T07:15:49,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:370730x0, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T07:15:49,981 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37073-0x1013833366e0000 connected 2024-11-14T07:15:50,059 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:15:50,062 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:15:50,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:15:50,075 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22, hbase.cluster.distributed=false 2024-11-14T07:15:50,095 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T07:15:50,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37073 2024-11-14T07:15:50,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37073 2024-11-14T07:15:50,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37073 2024-11-14T07:15:50,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37073 2024-11-14T07:15:50,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37073 2024-11-14T07:15:50,195 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c7aa6e2790b3:0 server-side Connection retries=45 2024-11-14T07:15:50,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:15:50,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T07:15:50,197 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T07:15:50,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:15:50,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T07:15:50,200 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T07:15:50,202 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T07:15:50,203 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33845 2024-11-14T07:15:50,205 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33845 connecting to ZooKeeper ensemble=127.0.0.1:58901 2024-11-14T07:15:50,206 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:15:50,209 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:15:50,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338450x0, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T07:15:50,227 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:338450x0, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:15:50,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33845-0x1013833366e0001 connected 2024-11-14T07:15:50,231 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T07:15:50,240 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T07:15:50,243 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T07:15:50,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T07:15:50,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33845 2024-11-14T07:15:50,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33845 2024-11-14T07:15:50,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33845 2024-11-14T07:15:50,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33845 2024-11-14T07:15:50,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33845 2024-11-14T07:15:50,273 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c7aa6e2790b3:37073 2024-11-14T07:15:50,274 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:50,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:15:50,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:15:50,291 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:50,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:50,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T07:15:50,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:50,332 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T07:15:50,333 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c7aa6e2790b3,37073,1731568549456 from backup master directory 2024-11-14T07:15:50,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:15:50,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:50,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:15:50,423 WARN [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T07:15:50,423 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:50,427 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T07:15:50,429 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T07:15:50,485 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/hbase.id] with ID: 9a0774c9-082c-43e3-ac28-5f93eb53c921 2024-11-14T07:15:50,485 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/.tmp/hbase.id 2024-11-14T07:15:50,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741826_1002 (size=42) 2024-11-14T07:15:50,900 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/.tmp/hbase.id]:[hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/hbase.id] 2024-11-14T07:15:50,948 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:15:50,953 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T07:15:50,977 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-14T07:15:51,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:51,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741827_1003 (size=196) 2024-11-14T07:15:51,078 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:15:51,081 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T07:15:51,098 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:51,103 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T07:15:51,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741828_1004 (size=1189) 2024-11-14T07:15:51,573 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store 2024-11-14T07:15:51,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741829_1005 (size=34) 2024-11-14T07:15:51,608 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T07:15:51,612 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:15:51,614 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T07:15:51,614 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:15:51,615 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:15:51,617 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T07:15:51,617 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:15:51,617 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:15:51,619 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731568551614Disabling compacts and flushes for region at 1731568551614Disabling writes for close at 1731568551617 (+3 ms)Writing region close event to WAL at 1731568551617Closed at 1731568551617 2024-11-14T07:15:51,622 WARN [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/.initializing 2024-11-14T07:15:51,622 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/WALs/c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:51,634 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T07:15:51,655 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c7aa6e2790b3%2C37073%2C1731568549456, suffix=, logDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/WALs/c7aa6e2790b3,37073,1731568549456, archiveDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/oldWALs, maxLogs=10 2024-11-14T07:15:51,685 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/WALs/c7aa6e2790b3,37073,1731568549456/c7aa6e2790b3%2C37073%2C1731568549456.1731568551660, exclude list is [], retry=0 2024-11-14T07:15:51,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33971,DS-bbe5f45d-0a0e-4dc5-ac16-fa0b5dce602f,DISK] 2024-11-14T07:15:51,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T07:15:51,750 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/WALs/c7aa6e2790b3,37073,1731568549456/c7aa6e2790b3%2C37073%2C1731568549456.1731568551660 2024-11-14T07:15:51,751 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37215:37215)] 2024-11-14T07:15:51,751 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:15:51,752 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:15:51,755 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,756 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T07:15:51,838 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:51,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:51,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,845 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T07:15:51,846 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:51,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:15:51,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T07:15:51,851 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:51,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:15:51,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T07:15:51,856 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:51,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:15:51,859 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,864 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,866 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,872 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,873 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,878 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T07:15:51,891 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:15:51,897 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:15:51,899 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67628384, jitterRate=0.007741451263427734}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T07:15:51,906 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731568551774Initializing all the Stores at 1731568551777 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731568551778 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568551779 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568551780 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568551780Cleaning up temporary data from old regions at 1731568551874 (+94 ms)Region opened successfully at 1731568551906 (+32 ms) 2024-11-14T07:15:51,908 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T07:15:51,953 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d774ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c7aa6e2790b3/172.17.0.2:0 2024-11-14T07:15:51,995 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T07:15:52,008 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T07:15:52,009 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T07:15:52,012 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T07:15:52,014 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T07:15:52,020 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T07:15:52,020 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T07:15:52,058 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T07:15:52,070 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T07:15:52,110 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T07:15:52,113 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T07:15:52,115 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T07:15:52,120 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T07:15:52,123 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T07:15:52,128 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T07:15:52,140 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T07:15:52,143 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T07:15:52,152 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T07:15:52,175 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T07:15:52,183 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T07:15:52,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T07:15:52,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T07:15:52,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:52,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:52,197 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c7aa6e2790b3,37073,1731568549456, sessionid=0x1013833366e0000, setting cluster-up flag (Was=false) 2024-11-14T07:15:52,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:52,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:52,257 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T07:15:52,259 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:52,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:52,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:52,310 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T07:15:52,312 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:52,319 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T07:15:52,371 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(746): ClusterId : 9a0774c9-082c-43e3-ac28-5f93eb53c921 2024-11-14T07:15:52,374 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T07:15:52,389 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T07:15:52,389 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T07:15:52,397 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T07:15:52,397 DEBUG [RS:0;c7aa6e2790b3:33845 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6122d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c7aa6e2790b3/172.17.0.2:0 2024-11-14T07:15:52,406 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T07:15:52,417 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c7aa6e2790b3:33845 2024-11-14T07:15:52,417 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T07:15:52,421 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T07:15:52,421 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T07:15:52,422 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T07:15:52,425 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(2659): reportForDuty to master=c7aa6e2790b3,37073,1731568549456 with port=33845, startcode=1731568550163 2024-11-14T07:15:52,425 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T07:15:52,432 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c7aa6e2790b3,37073,1731568549456 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T07:15:52,439 DEBUG [RS:0;c7aa6e2790b3:33845 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T07:15:52,442 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:15:52,442 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:15:52,443 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:15:52,443 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:15:52,443 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c7aa6e2790b3:0, corePoolSize=10, maxPoolSize=10 2024-11-14T07:15:52,444 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,444 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=2, maxPoolSize=2 2024-11-14T07:15:52,444 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,449 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731568582449 2024-11-14T07:15:52,452 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T07:15:52,453 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T07:15:52,453 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T07:15:52,453 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T07:15:52,457 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T07:15:52,457 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T07:15:52,458 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T07:15:52,458 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T07:15:52,461 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:52,461 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T07:15:52,461 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,470 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T07:15:52,472 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T07:15:52,472 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T07:15:52,476 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T07:15:52,477 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T07:15:52,480 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.large.0-1731568552478,5,FailOnTimeoutGroup] 2024-11-14T07:15:52,481 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.small.0-1731568552481,5,FailOnTimeoutGroup] 2024-11-14T07:15:52,481 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,482 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T07:15:52,490 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,491 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,491 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47465, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T07:15:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741831_1007 (size=1321) 2024-11-14T07:15:52,497 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T07:15:52,497 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:15:52,499 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37073 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:52,502 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37073 {}] master.ServerManager(517): Registering regionserver=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:52,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741832_1008 (size=32) 2024-11-14T07:15:52,520 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:15:52,520 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39657 2024-11-14T07:15:52,520 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T07:15:52,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T07:15:52,532 DEBUG [RS:0;c7aa6e2790b3:33845 {}] zookeeper.ZKUtil(111): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:52,533 WARN [RS:0;c7aa6e2790b3:33845 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T07:15:52,533 INFO [RS:0;c7aa6e2790b3:33845 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T07:15:52,533 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/WALs/c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:52,535 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c7aa6e2790b3,33845,1731568550163] 2024-11-14T07:15:52,566 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T07:15:52,589 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T07:15:52,596 INFO [RS:0;c7aa6e2790b3:33845 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T07:15:52,596 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,598 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T07:15:52,605 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T07:15:52,608 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,608 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,608 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,608 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,609 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,609 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,609 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c7aa6e2790b3:0, corePoolSize=2, maxPoolSize=2 2024-11-14T07:15:52,609 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,609 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,610 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,610 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,610 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,610 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:15:52,610 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c7aa6e2790b3:0, corePoolSize=3, maxPoolSize=3 2024-11-14T07:15:52,611 DEBUG [RS:0;c7aa6e2790b3:33845 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0, corePoolSize=3, maxPoolSize=3 2024-11-14T07:15:52,614 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,615 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,615 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,615 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,615 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,615 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,33845,1731568550163-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T07:15:52,638 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T07:15:52,639 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,33845,1731568550163-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,640 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,640 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.Replication(171): c7aa6e2790b3,33845,1731568550163 started 2024-11-14T07:15:52,661 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:52,661 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1482): Serving as c7aa6e2790b3,33845,1731568550163, RpcServer on c7aa6e2790b3/172.17.0.2:33845, sessionid=0x1013833366e0001 2024-11-14T07:15:52,662 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T07:15:52,662 DEBUG [RS:0;c7aa6e2790b3:33845 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:52,662 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c7aa6e2790b3,33845,1731568550163' 2024-11-14T07:15:52,662 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T07:15:52,664 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T07:15:52,664 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T07:15:52,664 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T07:15:52,665 DEBUG [RS:0;c7aa6e2790b3:33845 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:52,665 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c7aa6e2790b3,33845,1731568550163' 2024-11-14T07:15:52,665 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T07:15:52,666 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T07:15:52,666 DEBUG [RS:0;c7aa6e2790b3:33845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T07:15:52,666 INFO [RS:0;c7aa6e2790b3:33845 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T07:15:52,666 INFO [RS:0;c7aa6e2790b3:33845 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T07:15:52,773 INFO [RS:0;c7aa6e2790b3:33845 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T07:15:52,776 INFO [RS:0;c7aa6e2790b3:33845 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c7aa6e2790b3%2C33845%2C1731568550163, suffix=, logDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/WALs/c7aa6e2790b3,33845,1731568550163, archiveDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/oldWALs, maxLogs=32 2024-11-14T07:15:52,791 DEBUG [RS:0;c7aa6e2790b3:33845 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/WALs/c7aa6e2790b3,33845,1731568550163/c7aa6e2790b3%2C33845%2C1731568550163.1731568552780, exclude list is [], retry=0 2024-11-14T07:15:52,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33971,DS-bbe5f45d-0a0e-4dc5-ac16-fa0b5dce602f,DISK] 2024-11-14T07:15:52,799 INFO [RS:0;c7aa6e2790b3:33845 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/WALs/c7aa6e2790b3,33845,1731568550163/c7aa6e2790b3%2C33845%2C1731568550163.1731568552780 2024-11-14T07:15:52,799 DEBUG [RS:0;c7aa6e2790b3:33845 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37215:37215)] 2024-11-14T07:15:52,920 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:15:52,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T07:15:52,927 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T07:15:52,927 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:52,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:52,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T07:15:52,931 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T07:15:52,931 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:52,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:52,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T07:15:52,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T07:15:52,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:52,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:52,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T07:15:52,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T07:15:52,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:52,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:52,940 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T07:15:52,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740 2024-11-14T07:15:52,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740 2024-11-14T07:15:52,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T07:15:52,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T07:15:52,948 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:15:52,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T07:15:52,956 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:15:52,957 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64778833, jitterRate=-0.03472016751766205}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:15:52,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731568552920Initializing all the Stores at 1731568552922 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731568552923 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731568552923Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568552923Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731568552923Cleaning up temporary data from old regions at 1731568552947 (+24 ms)Region opened successfully at 1731568552960 (+13 ms) 2024-11-14T07:15:52,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T07:15:52,960 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T07:15:52,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T07:15:52,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T07:15:52,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T07:15:52,962 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T07:15:52,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731568552960Disabling compacts and flushes for region at 1731568552960Disabling writes for close at 1731568552960Writing region close event to WAL at 1731568552961 (+1 ms)Closed at 1731568552962 (+1 ms) 2024-11-14T07:15:52,965 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T07:15:52,965 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T07:15:52,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T07:15:52,978 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T07:15:52,981 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T07:15:53,133 DEBUG [c7aa6e2790b3:37073 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T07:15:53,145 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:53,151 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c7aa6e2790b3,33845,1731568550163, state=OPENING 2024-11-14T07:15:53,173 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T07:15:53,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:53,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:15:53,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:15:53,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:15:53,186 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T07:15:53,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:15:53,362 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T07:15:53,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58659, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T07:15:53,377 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T07:15:53,377 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T07:15:53,377 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T07:15:53,381 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c7aa6e2790b3%2C33845%2C1731568550163.meta, suffix=.meta, logDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/WALs/c7aa6e2790b3,33845,1731568550163, archiveDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/oldWALs, maxLogs=32 2024-11-14T07:15:53,399 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/WALs/c7aa6e2790b3,33845,1731568550163/c7aa6e2790b3%2C33845%2C1731568550163.meta.1731568553383.meta, exclude list is [], retry=0 2024-11-14T07:15:53,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33971,DS-bbe5f45d-0a0e-4dc5-ac16-fa0b5dce602f,DISK] 2024-11-14T07:15:53,408 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/WALs/c7aa6e2790b3,33845,1731568550163/c7aa6e2790b3%2C33845%2C1731568550163.meta.1731568553383.meta 2024-11-14T07:15:53,412 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37215:37215)] 2024-11-14T07:15:53,412 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:15:53,415 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T07:15:53,418 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T07:15:53,424 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T07:15:53,430 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T07:15:53,431 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:15:53,431 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T07:15:53,431 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T07:15:53,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T07:15:53,437 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T07:15:53,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:53,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:53,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T07:15:53,440 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T07:15:53,440 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:53,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:53,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T07:15:53,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T07:15:53,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:53,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:53,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T07:15:53,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T07:15:53,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:53,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:15:53,448 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T07:15:53,450 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740 2024-11-14T07:15:53,454 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740 2024-11-14T07:15:53,457 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T07:15:53,457 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T07:15:53,458 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:15:53,465 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T07:15:53,468 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66661388, jitterRate=-0.006667912006378174}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:15:53,468 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T07:15:53,470 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731568553432Writing region info on filesystem at 1731568553432Initializing all the Stores at 1731568553434 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731568553434Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731568553435 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568553435Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731568553435Cleaning up temporary data from old regions at 1731568553457 (+22 ms)Running coprocessor post-open hooks at 1731568553468 (+11 ms)Region opened successfully at 1731568553470 (+2 ms) 2024-11-14T07:15:53,479 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731568553354 2024-11-14T07:15:53,494 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T07:15:53,495 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T07:15:53,497 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:53,500 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c7aa6e2790b3,33845,1731568550163, state=OPEN 2024-11-14T07:15:53,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T07:15:53,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T07:15:53,540 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:15:53,540 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:15:53,542 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:53,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T07:15:53,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c7aa6e2790b3,33845,1731568550163 in 354 msec 2024-11-14T07:15:53,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T07:15:53,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 581 msec 2024-11-14T07:15:53,561 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T07:15:53,561 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T07:15:53,581 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:53,583 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:53,611 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:53,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37961, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:53,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2840 sec 2024-11-14T07:15:53,637 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731568553637, completionTime=-1 2024-11-14T07:15:53,640 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T07:15:53,640 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T07:15:53,670 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T07:15:53,670 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731568613670 2024-11-14T07:15:53,670 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731568673670 2024-11-14T07:15:53,670 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-14T07:15:53,673 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,37073,1731568549456-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:53,674 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,37073,1731568549456-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:53,674 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,37073,1731568549456-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:53,676 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c7aa6e2790b3:37073, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:53,682 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:53,684 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:53,688 DEBUG [master/c7aa6e2790b3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T07:15:53,717 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.293sec 2024-11-14T07:15:53,719 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T07:15:53,720 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T07:15:53,721 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T07:15:53,721 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T07:15:53,722 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T07:15:53,722 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,37073,1731568549456-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T07:15:53,723 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,37073,1731568549456-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T07:15:53,730 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T07:15:53,731 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T07:15:53,731 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,37073,1731568549456-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:15:53,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:53,782 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T07:15:53,782 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T07:15:53,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:53,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:53,794 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:53,804 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:53,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:53,808 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:53,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:53,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:53,852 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=7158 2024-11-14T07:15:53,862 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:15:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-14T07:15:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:15:53,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:15:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:15:53,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:15:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:15:53,910 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:15:53,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T07:15:53,913 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:53,919 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:15:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:15:53,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741835_1011 (size=1061) 2024-11-14T07:15:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:15:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:15:54,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:15:54,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741836_1012 (size=53) 2024-11-14T07:15:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:15:54,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:15:54,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 349b5cba99ebf7c21762f59b06a8ec03, disabling compactions & flushes 2024-11-14T07:15:54,748 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:54,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:54,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. after waiting 0 ms 2024-11-14T07:15:54,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:54,749 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:54,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 349b5cba99ebf7c21762f59b06a8ec03: Waiting for close lock at 1731568554748Disabling compacts and flushes for region at 1731568554748Disabling writes for close at 1731568554748Writing region close event to WAL at 1731568554749 (+1 ms)Closed at 1731568554749 2024-11-14T07:15:54,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:15:54,758 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731568554752"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731568554752"}]},"ts":"1731568554752"} 2024-11-14T07:15:54,765 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:15:54,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:15:54,769 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568554767"}]},"ts":"1731568554767"} 2024-11-14T07:15:54,774 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:15:54,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, ASSIGN}] 2024-11-14T07:15:54,779 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, ASSIGN 2024-11-14T07:15:54,781 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=false 2024-11-14T07:15:54,933 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=349b5cba99ebf7c21762f59b06a8ec03, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:54,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, ASSIGN because future has completed 2024-11-14T07:15:54,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:15:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:15:55,110 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:55,110 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:15:55,110 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,110 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:15:55,110 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,110 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,113 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,116 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:15:55,116 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 349b5cba99ebf7c21762f59b06a8ec03 columnFamilyName A 2024-11-14T07:15:55,116 DEBUG [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:55,118 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.HStore(327): Store=349b5cba99ebf7c21762f59b06a8ec03/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:15:55,118 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,120 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:15:55,120 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 349b5cba99ebf7c21762f59b06a8ec03 columnFamilyName B 2024-11-14T07:15:55,121 DEBUG [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:55,121 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.HStore(327): Store=349b5cba99ebf7c21762f59b06a8ec03/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:15:55,122 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,124 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:15:55,124 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 349b5cba99ebf7c21762f59b06a8ec03 columnFamilyName C 2024-11-14T07:15:55,124 DEBUG [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:15:55,125 INFO [StoreOpener-349b5cba99ebf7c21762f59b06a8ec03-1 {}] regionserver.HStore(327): Store=349b5cba99ebf7c21762f59b06a8ec03/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:15:55,126 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:55,126 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,127 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,128 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,130 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,130 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,131 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:15:55,133 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,137 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:15:55,137 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 349b5cba99ebf7c21762f59b06a8ec03; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63568928, jitterRate=-0.0527491569519043}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:15:55,138 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:55,138 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 349b5cba99ebf7c21762f59b06a8ec03: Running coprocessor pre-open hook at 1731568555111Writing region info on filesystem at 1731568555111Initializing all the Stores at 1731568555112 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568555112Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568555112Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568555112Cleaning up temporary data from old regions at 1731568555130 (+18 ms)Running coprocessor post-open hooks at 1731568555138 (+8 ms)Region opened successfully at 1731568555138 2024-11-14T07:15:55,140 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., pid=6, masterSystemTime=1731568555098 2024-11-14T07:15:55,144 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:55,144 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:55,145 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=349b5cba99ebf7c21762f59b06a8ec03, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:55,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:15:55,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T07:15:55,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 in 209 msec 2024-11-14T07:15:55,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T07:15:55,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, ASSIGN in 377 msec 2024-11-14T07:15:55,158 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:15:55,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568555158"}]},"ts":"1731568555158"} 2024-11-14T07:15:55,162 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:15:55,163 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:15:55,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2610 sec 2024-11-14T07:15:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:15:56,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:15:56,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,088 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d58b5fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,091 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af0af03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,093 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:15:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,37073,1731568549456 2024-11-14T07:15:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3498947e 2024-11-14T07:15:56,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:15:56,106 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:15:56,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,112 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,115 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,117 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,118 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,121 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,125 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5970e36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,136 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2199d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,139 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,141 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,145 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,148 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129161b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d030762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365ce344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,157 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a1a0ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,160 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee9f900, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:15:56,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:15:56,163 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:15:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:15:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:15:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1608b68b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:15:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:15:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:15:56,165 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:15:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e186e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:15:56,170 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,170 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,170 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,171 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,171 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,171 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,173 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,182 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,185 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:15:56,189 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,189 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:15:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,189 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,189 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:15:56,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:15:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:15:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T07:15:56,215 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:15:56,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:15:56,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:15:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:15:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:56,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:15:56,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:15:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:15:56,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:56,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:15:56,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:15:56,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:15:56,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568616410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568616417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568616430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568616435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568616441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ce533cced4df44ee877b9d66b2e0f90b is 50, key is test_row_0/A:col10/1731568556223/Put/seqid=0 2024-11-14T07:15:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741837_1013 (size=12001) 2024-11-14T07:15:56,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ce533cced4df44ee877b9d66b2e0f90b 2024-11-14T07:15:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:15:56,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:15:56,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568616605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568616605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568616606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568616607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568616606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/a902e75d68be43aabe8dfb52527149db is 50, key is test_row_0/B:col10/1731568556223/Put/seqid=0 2024-11-14T07:15:56,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741838_1014 (size=12001) 2024-11-14T07:15:56,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/a902e75d68be43aabe8dfb52527149db 2024-11-14T07:15:56,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/cf2efe853cfe41a5b7ec2a335b27ac43 is 50, key is test_row_0/C:col10/1731568556223/Put/seqid=0 2024-11-14T07:15:56,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741839_1015 (size=12001) 2024-11-14T07:15:56,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:15:56,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:56,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568616823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568616823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568616823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568616834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568616834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:15:56,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:15:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:56,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:57,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:15:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:57,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568617133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568617133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:57,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568617133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:57,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568617143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:57,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568617154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/cf2efe853cfe41a5b7ec2a335b27ac43 2024-11-14T07:15:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:57,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ce533cced4df44ee877b9d66b2e0f90b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ce533cced4df44ee877b9d66b2e0f90b 2024-11-14T07:15:57,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ce533cced4df44ee877b9d66b2e0f90b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:15:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/a902e75d68be43aabe8dfb52527149db as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a902e75d68be43aabe8dfb52527149db 2024-11-14T07:15:57,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a902e75d68be43aabe8dfb52527149db, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:15:57,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/cf2efe853cfe41a5b7ec2a335b27ac43 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cf2efe853cfe41a5b7ec2a335b27ac43 2024-11-14T07:15:57,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:15:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:57,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:57,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cf2efe853cfe41a5b7ec2a335b27ac43, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:15:57,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 349b5cba99ebf7c21762f59b06a8ec03 in 975ms, sequenceid=13, compaction requested=false 2024-11-14T07:15:57,261 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T07:15:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:15:57,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:15:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:57,404 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:15:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:15:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:15:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:15:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:57,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/2aa64732fef54dcb834dc1934909bb83 is 50, key is test_row_0/A:col10/1731568556428/Put/seqid=0 2024-11-14T07:15:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741840_1016 (size=12001) 2024-11-14T07:15:57,466 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/2aa64732fef54dcb834dc1934909bb83 2024-11-14T07:15:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/63776d3e48354ecd9d5f5f4df3d8a5d2 is 50, key is test_row_0/B:col10/1731568556428/Put/seqid=0 2024-11-14T07:15:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741841_1017 (size=12001) 2024-11-14T07:15:57,522 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/63776d3e48354ecd9d5f5f4df3d8a5d2 2024-11-14T07:15:57,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f329b8302ba3493c9716cbd7824a52d8 is 50, key is test_row_0/C:col10/1731568556428/Put/seqid=0 2024-11-14T07:15:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741842_1018 (size=12001) 2024-11-14T07:15:57,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f329b8302ba3493c9716cbd7824a52d8 2024-11-14T07:15:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/2aa64732fef54dcb834dc1934909bb83 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2aa64732fef54dcb834dc1934909bb83 2024-11-14T07:15:57,617 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2aa64732fef54dcb834dc1934909bb83, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T07:15:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/63776d3e48354ecd9d5f5f4df3d8a5d2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63776d3e48354ecd9d5f5f4df3d8a5d2 2024-11-14T07:15:57,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63776d3e48354ecd9d5f5f4df3d8a5d2, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T07:15:57,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f329b8302ba3493c9716cbd7824a52d8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f329b8302ba3493c9716cbd7824a52d8 2024-11-14T07:15:57,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:57,674 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f329b8302ba3493c9716cbd7824a52d8, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T07:15:57,676 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for 349b5cba99ebf7c21762f59b06a8ec03 in 271ms, sequenceid=33, compaction requested=false 2024-11-14T07:15:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:15:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T07:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T07:15:57,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T07:15:57,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-14T07:15:57,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4820 sec 2024-11-14T07:15:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:57,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:15:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:15:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:15:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:15:57,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:15:57,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/5c4b7b3aa67b476fb7329b3521e52c88 is 50, key is test_row_0/A:col10/1731568557691/Put/seqid=0 2024-11-14T07:15:57,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741843_1019 (size=16681) 2024-11-14T07:15:57,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/5c4b7b3aa67b476fb7329b3521e52c88 2024-11-14T07:15:57,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/a09c13de5d3c4a0fa528702e1e5e47c5 is 50, key is test_row_0/B:col10/1731568557691/Put/seqid=0 2024-11-14T07:15:57,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741844_1020 (size=12001) 2024-11-14T07:15:57,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/a09c13de5d3c4a0fa528702e1e5e47c5 2024-11-14T07:15:57,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f53b757a2c85411789c82a5472a5e8f7 is 50, key is test_row_0/C:col10/1731568557691/Put/seqid=0 2024-11-14T07:15:57,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731568557833, max timestamp=1731568557852Num uniques 859; 2024-11-14T07:15:57,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T07:15:57,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568557833, max timestamp=1731568557856Num uniques 859; 2024-11-14T07:15:57,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:15:57,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568557833, max timestamp=1731568557855Num uniques 859; 2024-11-14T07:15:57,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:15:57,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741845_1021 (size=12001) 2024-11-14T07:15:57,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568617940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:57,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568617943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:57,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:57,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568557868, max timestamp=1731568557954Num uniques 1009; 2024-11-14T07:15:57,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-14T07:15:57,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568557868, max timestamp=1731568557954Num uniques 1018; 2024-11-14T07:15:57,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T07:15:57,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568557868, max timestamp=1731568557954Num uniques 1018; 2024-11-14T07:15:57,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T07:15:58,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568618064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568618066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568618072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568618073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568618073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568618183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568618183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568618192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568618293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568618299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f53b757a2c85411789c82a5472a5e8f7 2024-11-14T07:15:58,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/5c4b7b3aa67b476fb7329b3521e52c88 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5c4b7b3aa67b476fb7329b3521e52c88 2024-11-14T07:15:58,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5c4b7b3aa67b476fb7329b3521e52c88, entries=250, sequenceid=43, filesize=16.3 K 2024-11-14T07:15:58,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/a09c13de5d3c4a0fa528702e1e5e47c5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a09c13de5d3c4a0fa528702e1e5e47c5 2024-11-14T07:15:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:15:58,373 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:15:58,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:15:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T07:15:58,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a09c13de5d3c4a0fa528702e1e5e47c5, entries=150, sequenceid=43, filesize=11.7 K 2024-11-14T07:15:58,377 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:15:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:15:58,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f53b757a2c85411789c82a5472a5e8f7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f53b757a2c85411789c82a5472a5e8f7 2024-11-14T07:15:58,379 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:15:58,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:15:58,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f53b757a2c85411789c82a5472a5e8f7, entries=150, sequenceid=43, filesize=11.7 K 2024-11-14T07:15:58,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 349b5cba99ebf7c21762f59b06a8ec03 in 703ms, sequenceid=43, compaction requested=true 2024-11-14T07:15:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:15:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:15:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:15:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:15:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:15:58,427 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:15:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:15:58,427 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:15:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:15:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:15:58,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=140.89 KB heapSize=399.84 KB 2024-11-14T07:15:58,434 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:15:58,436 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:15:58,436 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:15:58,436 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:15:58,437 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,438 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a902e75d68be43aabe8dfb52527149db, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63776d3e48354ecd9d5f5f4df3d8a5d2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a09c13de5d3c4a0fa528702e1e5e47c5] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=35.2 K 2024-11-14T07:15:58,439 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,440 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a902e75d68be43aabe8dfb52527149db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731568556223 2024-11-14T07:15:58,442 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 63776d3e48354ecd9d5f5f4df3d8a5d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731568556351 2024-11-14T07:15:58,443 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a09c13de5d3c4a0fa528702e1e5e47c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731568557676 2024-11-14T07:15:58,444 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ce533cced4df44ee877b9d66b2e0f90b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2aa64732fef54dcb834dc1934909bb83, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5c4b7b3aa67b476fb7329b3521e52c88] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=39.7 K 2024-11-14T07:15:58,447 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce533cced4df44ee877b9d66b2e0f90b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731568556223 2024-11-14T07:15:58,448 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa64732fef54dcb834dc1934909bb83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731568556351 2024-11-14T07:15:58,449 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4b7b3aa67b476fb7329b3521e52c88, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731568557660 2024-11-14T07:15:58,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731568558034, max timestamp=1731568558417Num uniques 1077; 2024-11-14T07:15:58,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T07:15:58,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731568558034, max timestamp=1731568558409Num uniques 1077; 2024-11-14T07:15:58,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T07:15:58,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:15:58,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:15:58,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:15:58,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:15:58,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:15:58,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731568558034, max timestamp=1731568558420Num uniques 1077; 2024-11-14T07:15:58,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:15:58,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/5503aecb8ba34fe7af5ef96c2f110644 is 50, key is test_row_0/A:col10/1731568558417/Put/seqid=0 2024-11-14T07:15:58,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T07:15:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:15:58,501 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:15:58,502 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/c9c1f9fb81234c10aa9fce0fd899a008 is 50, key is test_row_0/A:col10/1731568557691/Put/seqid=0 2024-11-14T07:15:58,517 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:15:58,518 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/bb42e219d6d349ca8934a70d2b072fbb is 50, key is test_row_0/B:col10/1731568557691/Put/seqid=0 2024-11-14T07:15:58,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741846_1022 (size=14341) 2024-11-14T07:15:58,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/5503aecb8ba34fe7af5ef96c2f110644 2024-11-14T07:15:58,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741847_1023 (size=12104) 2024-11-14T07:15:58,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741848_1024 (size=12104) 2024-11-14T07:15:58,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/5d86c2d8ebb940248af19bb717e28c03 is 50, key is test_row_0/B:col10/1731568558417/Put/seqid=0 2024-11-14T07:15:58,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568558555, max timestamp=1731568558593Num uniques 859; 2024-11-14T07:15:58,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:15:58,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568558555, max timestamp=1731568558593Num uniques 859; 2024-11-14T07:15:58,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:15:58,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568558555, max timestamp=1731568558593Num uniques 859; 2024-11-14T07:15:58,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:15:58,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741849_1025 (size=12001) 2024-11-14T07:15:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:15:58,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:58,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:58,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568618694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731568558690, max timestamp=1731568558706Num uniques 1018; 2024-11-14T07:15:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7685, totalHeapSize=17712, min timestamp=1731568558690, max timestamp=1731568558706Num uniques 1018; 2024-11-14T07:15:58,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T07:15:58,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46615, new segment data size=7685, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T07:15:58,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7682, totalHeapSize=17712, min timestamp=1731568558690, max timestamp=1731568558706Num uniques 1018; 2024-11-14T07:15:58,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46612, new segment data size=7682, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T07:15:58,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731568558755, max timestamp=1731568558793Num uniques 1027; 2024-11-14T07:15:58,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-14T07:15:58,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731568558755, max timestamp=1731568558793Num uniques 1027; 2024-11-14T07:15:58,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-14T07:15:58,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731568558755, max timestamp=1731568558793Num uniques 1027; 2024-11-14T07:15:58,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-14T07:15:58,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:58,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:58,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:58,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:58,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568618864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568618871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568618871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568618875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568618878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,956 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:15:58,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568618982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,990 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/bb42e219d6d349ca8934a70d2b072fbb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/bb42e219d6d349ca8934a70d2b072fbb 2024-11-14T07:15:58,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568618993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568618993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:58,997 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/c9c1f9fb81234c10aa9fce0fd899a008 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/c9c1f9fb81234c10aa9fce0fd899a008 2024-11-14T07:15:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:15:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568618994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568618995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/5d86c2d8ebb940248af19bb717e28c03 2024-11-14T07:15:59,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,039 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into c9c1f9fb81234c10aa9fce0fd899a008(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:15:59,040 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:15:59,041 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into bb42e219d6d349ca8934a70d2b072fbb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:15:59,041 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:15:59,046 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=13, startTime=1731568558405; duration=0sec 2024-11-14T07:15:59,046 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=13, startTime=1731568558427; duration=0sec 2024-11-14T07:15:59,047 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:15:59,047 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:15:59,047 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:15:59,047 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:15:59,047 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:15:59,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d360389e38cb4942a20cc3428f0d4761 is 50, key is test_row_0/C:col10/1731568558417/Put/seqid=0 2024-11-14T07:15:59,054 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:15:59,055 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:15:59,055 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,055 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cf2efe853cfe41a5b7ec2a335b27ac43, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f329b8302ba3493c9716cbd7824a52d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f53b757a2c85411789c82a5472a5e8f7] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=35.2 K 2024-11-14T07:15:59,056 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2efe853cfe41a5b7ec2a335b27ac43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731568556223 2024-11-14T07:15:59,057 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f329b8302ba3493c9716cbd7824a52d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731568556351 2024-11-14T07:15:59,058 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f53b757a2c85411789c82a5472a5e8f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731568557676 2024-11-14T07:15:59,078 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T07:15:59,090 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:15:59,091 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/048873d7f55047808f819454832fa7ce is 50, key is test_row_0/C:col10/1731568557691/Put/seqid=0 2024-11-14T07:15:59,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741850_1026 (size=12001) 2024-11-14T07:15:59,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741851_1027 (size=12104) 2024-11-14T07:15:59,124 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/048873d7f55047808f819454832fa7ce as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/048873d7f55047808f819454832fa7ce 2024-11-14T07:15:59,139 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into 048873d7f55047808f819454832fa7ce(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:15:59,139 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:15:59,139 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=13, startTime=1731568558427; duration=0sec 2024-11-14T07:15:59,139 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:15:59,139 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:15:59,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:59,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:59,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568619193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568619202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568619202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568619212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568619223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:59,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:59,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:59,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:15:59,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:15:59,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d360389e38cb4942a20cc3428f0d4761 2024-11-14T07:15:59,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568619502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:15:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568619513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568619513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/5503aecb8ba34fe7af5ef96c2f110644 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5503aecb8ba34fe7af5ef96c2f110644 2024-11-14T07:15:59,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568619523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568619532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:15:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:15:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:15:59,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5503aecb8ba34fe7af5ef96c2f110644, entries=200, sequenceid=100, filesize=14.0 K 2024-11-14T07:15:59,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/5d86c2d8ebb940248af19bb717e28c03 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5d86c2d8ebb940248af19bb717e28c03 2024-11-14T07:15:59,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5d86c2d8ebb940248af19bb717e28c03, entries=150, sequenceid=100, filesize=11.7 K 2024-11-14T07:15:59,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d360389e38cb4942a20cc3428f0d4761 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d360389e38cb4942a20cc3428f0d4761 2024-11-14T07:15:59,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d360389e38cb4942a20cc3428f0d4761, entries=150, sequenceid=100, filesize=11.7 K 2024-11-14T07:15:59,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~104.34 KB/106848, currentSize=147.60 KB/151140 for 349b5cba99ebf7c21762f59b06a8ec03 in 1162ms, sequenceid=100, compaction requested=false 2024-11-14T07:15:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:15:59,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:15:59,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:15:59,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=147.60 KB heapSize=413.77 KB 2024-11-14T07:15:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:15:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:15:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:15:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:15:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:15:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:15:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/b9d2fe3fa4aa4a09a0936950a46544bc is 50, key is test_row_0/A:col10/1731568558860/Put/seqid=0 2024-11-14T07:15:59,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741852_1028 (size=12151) 2024-11-14T07:15:59,802 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T07:15:59,802 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T07:15:59,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T07:15:59,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T07:15:59,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T07:15:59,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T07:15:59,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T07:15:59,808 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T07:16:00,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:00,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568620032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568620035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568620038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568620042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568620052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,124 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/b9d2fe3fa4aa4a09a0936950a46544bc 2024-11-14T07:16:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568620143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/4e3a6781c2084374b947459cb7cb70f6 is 50, key is test_row_0/B:col10/1731568558860/Put/seqid=0 2024-11-14T07:16:00,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568620156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568620156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741853_1029 (size=12151) 2024-11-14T07:16:00,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/4e3a6781c2084374b947459cb7cb70f6 2024-11-14T07:16:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/10f2e80afc1c41b0920d0b59a293f182 is 50, key is test_row_0/C:col10/1731568558860/Put/seqid=0 2024-11-14T07:16:00,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741854_1030 (size=12151) 2024-11-14T07:16:00,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/10f2e80afc1c41b0920d0b59a293f182 2024-11-14T07:16:00,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/b9d2fe3fa4aa4a09a0936950a46544bc as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b9d2fe3fa4aa4a09a0936950a46544bc 2024-11-14T07:16:00,277 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b9d2fe3fa4aa4a09a0936950a46544bc, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T07:16:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/4e3a6781c2084374b947459cb7cb70f6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/4e3a6781c2084374b947459cb7cb70f6 2024-11-14T07:16:00,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/4e3a6781c2084374b947459cb7cb70f6, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T07:16:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/10f2e80afc1c41b0920d0b59a293f182 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/10f2e80afc1c41b0920d0b59a293f182 2024-11-14T07:16:00,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/10f2e80afc1c41b0920d0b59a293f182, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T07:16:00,321 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~413.72 KB/423648, currentSize=33.54 KB/34350 for 349b5cba99ebf7c21762f59b06a8ec03 in 675ms, sequenceid=174, compaction requested=true 2024-11-14T07:16:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T07:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T07:16:00,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T07:16:00,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9540 sec 2024-11-14T07:16:00,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9720 sec 2024-11-14T07:16:00,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:00,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:00,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:00,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:00,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:00,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:00,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:00,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/305b80bd54f64de4b53835f1dda980f9 is 50, key is test_row_0/A:col10/1731568560361/Put/seqid=0 2024-11-14T07:16:00,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741855_1031 (size=14541) 2024-11-14T07:16:00,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568560411, max timestamp=1731568560433Num uniques 859; 2024-11-14T07:16:00,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568560411, max timestamp=1731568560433Num uniques 859; 2024-11-14T07:16:00,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568560411, max timestamp=1731568560433Num uniques 859; 2024-11-14T07:16:00,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:00,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:00,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:00,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568620475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568620477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568620485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:16:00,522 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:00,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T07:16:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:16:00,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:00,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:00,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568620593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568620593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568620602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:16:00,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:00,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:00,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:00,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568620802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568620802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/305b80bd54f64de4b53835f1dda980f9 2024-11-14T07:16:00,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568620812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:00,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/66f30df076844addadd6ca55680c50c8 is 50, key is test_row_0/B:col10/1731568560361/Put/seqid=0 2024-11-14T07:16:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:16:00,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:00,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:00,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741856_1032 (size=12151) 2024-11-14T07:16:00,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/66f30df076844addadd6ca55680c50c8 2024-11-14T07:16:00,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/0a98be88cda94e988071b761f36a2ae4 is 50, key is test_row_0/C:col10/1731568560361/Put/seqid=0 2024-11-14T07:16:00,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741857_1033 (size=12151) 2024-11-14T07:16:00,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/0a98be88cda94e988071b761f36a2ae4 2024-11-14T07:16:00,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/305b80bd54f64de4b53835f1dda980f9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/305b80bd54f64de4b53835f1dda980f9 2024-11-14T07:16:00,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/305b80bd54f64de4b53835f1dda980f9, entries=200, sequenceid=184, filesize=14.2 K 2024-11-14T07:16:00,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/66f30df076844addadd6ca55680c50c8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/66f30df076844addadd6ca55680c50c8 2024-11-14T07:16:01,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/66f30df076844addadd6ca55680c50c8, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T07:16:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/0a98be88cda94e988071b761f36a2ae4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/0a98be88cda94e988071b761f36a2ae4 2024-11-14T07:16:01,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/0a98be88cda94e988071b761f36a2ae4, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T07:16:01,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 349b5cba99ebf7c21762f59b06a8ec03 in 676ms, sequenceid=184, compaction requested=true 2024-11-14T07:16:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:01,041 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:01,044 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53137 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:01,044 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:01,044 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,045 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/c9c1f9fb81234c10aa9fce0fd899a008, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5503aecb8ba34fe7af5ef96c2f110644, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b9d2fe3fa4aa4a09a0936950a46544bc, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/305b80bd54f64de4b53835f1dda980f9] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=51.9 K 2024-11-14T07:16:01,045 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c1f9fb81234c10aa9fce0fd899a008, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731568557676 2024-11-14T07:16:01,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:01,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:01,046 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:01,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:01,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:01,046 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5503aecb8ba34fe7af5ef96c2f110644, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731568558069 2024-11-14T07:16:01,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:01,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:01,047 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9d2fe3fa4aa4a09a0936950a46544bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731568558860 2024-11-14T07:16:01,048 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 305b80bd54f64de4b53835f1dda980f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568560028 2024-11-14T07:16:01,049 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:01,049 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:01,050 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,050 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/bb42e219d6d349ca8934a70d2b072fbb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5d86c2d8ebb940248af19bb717e28c03, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/4e3a6781c2084374b947459cb7cb70f6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/66f30df076844addadd6ca55680c50c8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=47.3 K 2024-11-14T07:16:01,053 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting bb42e219d6d349ca8934a70d2b072fbb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731568557676 2024-11-14T07:16:01,054 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d86c2d8ebb940248af19bb717e28c03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731568558409 2024-11-14T07:16:01,056 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e3a6781c2084374b947459cb7cb70f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731568558860 2024-11-14T07:16:01,058 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f30df076844addadd6ca55680c50c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568560028 2024-11-14T07:16:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:01,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-14T07:16:01,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:01,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:01,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:01,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:01,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:01,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:01,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/6d775e00fb9541a1b7d86ca7a98955ea is 50, key is test_row_0/A:col10/1731568561068/Put/seqid=0 2024-11-14T07:16:01,093 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:01,094 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/40de5dff305b4cba92d1c8052a39e024 is 50, key is test_row_0/B:col10/1731568560361/Put/seqid=0 2024-11-14T07:16:01,097 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#22 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:01,100 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/d486d2d9d59f4b41937cb9c4caef4c14 is 50, key is test_row_0/A:col10/1731568560361/Put/seqid=0 2024-11-14T07:16:01,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741858_1034 (size=14541) 2024-11-14T07:16:01,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741859_1035 (size=12391) 2024-11-14T07:16:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568621120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568621123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568621123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,131 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/40de5dff305b4cba92d1c8052a39e024 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/40de5dff305b4cba92d1c8052a39e024 2024-11-14T07:16:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568621123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568621127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,143 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into 40de5dff305b4cba92d1c8052a39e024(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:01,143 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:01,143 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=12, startTime=1731568561046; duration=0sec 2024-11-14T07:16:01,144 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:01,144 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:01,144 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:01,147 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:01,147 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:01,147 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,147 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/048873d7f55047808f819454832fa7ce, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d360389e38cb4942a20cc3428f0d4761, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/10f2e80afc1c41b0920d0b59a293f182, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/0a98be88cda94e988071b761f36a2ae4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=47.3 K 2024-11-14T07:16:01,148 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 048873d7f55047808f819454832fa7ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731568557676 2024-11-14T07:16:01,150 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d360389e38cb4942a20cc3428f0d4761, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731568558409 2024-11-14T07:16:01,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741860_1036 (size=12391) 2024-11-14T07:16:01,152 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 10f2e80afc1c41b0920d0b59a293f182, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731568558860 2024-11-14T07:16:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:16:01,153 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a98be88cda94e988071b761f36a2ae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568560028 2024-11-14T07:16:01,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,176 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:01,176 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/99b9f2b02a264a2aaca3cf6ee4c33bb5 is 50, key is test_row_0/C:col10/1731568560361/Put/seqid=0 2024-11-14T07:16:01,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741861_1037 (size=12391) 2024-11-14T07:16:01,200 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/99b9f2b02a264a2aaca3cf6ee4c33bb5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/99b9f2b02a264a2aaca3cf6ee4c33bb5 2024-11-14T07:16:01,212 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into 99b9f2b02a264a2aaca3cf6ee4c33bb5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:01,212 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:01,212 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=12, startTime=1731568561046; duration=0sec 2024-11-14T07:16:01,212 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:01,212 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568621232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568621242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568621252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:01,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:01,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568621442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568621452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568621461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/6d775e00fb9541a1b7d86ca7a98955ea 2024-11-14T07:16:01,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/757de47b84d44bf692b9238cd4fce127 is 50, key is test_row_0/B:col10/1731568561068/Put/seqid=0 2024-11-14T07:16:01,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741862_1038 (size=12151) 2024-11-14T07:16:01,566 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/d486d2d9d59f4b41937cb9c4caef4c14 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/d486d2d9d59f4b41937cb9c4caef4c14 2024-11-14T07:16:01,581 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into d486d2d9d59f4b41937cb9c4caef4c14(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:01,581 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:01,581 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=12, startTime=1731568561040; duration=0sec 2024-11-14T07:16:01,582 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:01,582 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:01,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:01,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:01,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568621642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568621642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:16:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568621752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568621762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568621772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:01,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:01,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:01,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/757de47b84d44bf692b9238cd4fce127 2024-11-14T07:16:01,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d270aa32447b4c2e823d4f257a3e89a5 is 50, key is test_row_0/C:col10/1731568561068/Put/seqid=0 2024-11-14T07:16:01,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741863_1039 (size=12151) 2024-11-14T07:16:02,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:02,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:02,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:02,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:02,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:02,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568622263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:02,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568622272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:02,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568622282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:02,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d270aa32447b4c2e823d4f257a3e89a5 2024-11-14T07:16:02,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/6d775e00fb9541a1b7d86ca7a98955ea as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6d775e00fb9541a1b7d86ca7a98955ea 2024-11-14T07:16:02,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:02,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:02,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:02,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:02,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6d775e00fb9541a1b7d86ca7a98955ea, entries=200, sequenceid=221, filesize=14.2 K 2024-11-14T07:16:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:02,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/757de47b84d44bf692b9238cd4fce127 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/757de47b84d44bf692b9238cd4fce127 2024-11-14T07:16:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/757de47b84d44bf692b9238cd4fce127, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T07:16:02,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d270aa32447b4c2e823d4f257a3e89a5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d270aa32447b4c2e823d4f257a3e89a5 2024-11-14T07:16:02,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d270aa32447b4c2e823d4f257a3e89a5, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T07:16:02,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 349b5cba99ebf7c21762f59b06a8ec03 in 1378ms, sequenceid=221, compaction requested=false 2024-11-14T07:16:02,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:02,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:16:02,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:02,558 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T07:16:02,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:02,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:02,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:02,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:02,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:02,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:02,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ebab44eeee5c4637ba17715af1c99c66 is 50, key is test_row_0/A:col10/1731568561118/Put/seqid=0 2024-11-14T07:16:02,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741864_1040 (size=12151) 2024-11-14T07:16:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:02,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:16:02,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568562685, max timestamp=1731568562727Num uniques 859; 2024-11-14T07:16:02,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568562685, max timestamp=1731568562727Num uniques 859; 2024-11-14T07:16:02,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568562685, max timestamp=1731568562727Num uniques 859; 2024-11-14T07:16:02,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:02,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:02,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:02,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568622781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:02,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:02,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568622785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:02,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568622891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:02,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568622902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:02,972 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ebab44eeee5c4637ba17715af1c99c66 2024-11-14T07:16:02,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/6253a463c3bb44cdb613fb92ecc165aa is 50, key is test_row_0/B:col10/1731568561118/Put/seqid=0 2024-11-14T07:16:03,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741865_1041 (size=12151) 2024-11-14T07:16:03,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/6253a463c3bb44cdb613fb92ecc165aa 2024-11-14T07:16:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a3424c78206549a0a288dbd3d9ba94a5 is 50, key is test_row_0/C:col10/1731568561118/Put/seqid=0 2024-11-14T07:16:03,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741866_1042 (size=12151) 2024-11-14T07:16:03,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568623102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568623113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568623282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568623282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,292 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:03,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,293 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568623302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,310 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568623412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568623432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a3424c78206549a0a288dbd3d9ba94a5 2024-11-14T07:16:03,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ebab44eeee5c4637ba17715af1c99c66 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ebab44eeee5c4637ba17715af1c99c66 2024-11-14T07:16:03,470 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ebab44eeee5c4637ba17715af1c99c66, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T07:16:03,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/6253a463c3bb44cdb613fb92ecc165aa as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6253a463c3bb44cdb613fb92ecc165aa 2024-11-14T07:16:03,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6253a463c3bb44cdb613fb92ecc165aa, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T07:16:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a3424c78206549a0a288dbd3d9ba94a5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a3424c78206549a0a288dbd3d9ba94a5 2024-11-14T07:16:03,499 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a3424c78206549a0a288dbd3d9ba94a5, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T07:16:03,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 349b5cba99ebf7c21762f59b06a8ec03 in 942ms, sequenceid=233, compaction requested=true 2024-11-14T07:16:03,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T07:16:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T07:16:03,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T07:16:03,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9760 sec 2024-11-14T07:16:03,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.9840 sec 2024-11-14T07:16:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:03,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-14T07:16:03,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:03,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:03,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/1eb5933fd17a4e61b312e5f1bfa272e7 is 50, key is test_row_0/A:col10/1731568563933/Put/seqid=0 2024-11-14T07:16:03,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741867_1043 (size=12301) 2024-11-14T07:16:03,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/1eb5933fd17a4e61b312e5f1bfa272e7 2024-11-14T07:16:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568623965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568623967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:03,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/2244f704dfef49e18474f580d5367215 is 50, key is test_row_0/B:col10/1731568563933/Put/seqid=0 2024-11-14T07:16:03,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741868_1044 (size=12301) 2024-11-14T07:16:03,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/2244f704dfef49e18474f580d5367215 2024-11-14T07:16:04,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/24552f08d61c478ba1f9150719cb773c is 50, key is test_row_0/C:col10/1731568563933/Put/seqid=0 2024-11-14T07:16:04,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741869_1045 (size=12301) 2024-11-14T07:16:04,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/24552f08d61c478ba1f9150719cb773c 2024-11-14T07:16:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/1eb5933fd17a4e61b312e5f1bfa272e7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/1eb5933fd17a4e61b312e5f1bfa272e7 2024-11-14T07:16:04,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568624082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/1eb5933fd17a4e61b312e5f1bfa272e7, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T07:16:04,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568624082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/2244f704dfef49e18474f580d5367215 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/2244f704dfef49e18474f580d5367215 2024-11-14T07:16:04,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/2244f704dfef49e18474f580d5367215, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T07:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/24552f08d61c478ba1f9150719cb773c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/24552f08d61c478ba1f9150719cb773c 2024-11-14T07:16:04,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/24552f08d61c478ba1f9150719cb773c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T07:16:04,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 349b5cba99ebf7c21762f59b06a8ec03 in 186ms, sequenceid=270, compaction requested=true 2024-11-14T07:16:04,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:04,122 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:04,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:04,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:04,122 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:04,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:04,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:04,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:04,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:04,125 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51384 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:04,125 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:04,125 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:04,125 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/d486d2d9d59f4b41937cb9c4caef4c14, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6d775e00fb9541a1b7d86ca7a98955ea, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ebab44eeee5c4637ba17715af1c99c66, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/1eb5933fd17a4e61b312e5f1bfa272e7] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=50.2 K 2024-11-14T07:16:04,126 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:04,126 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:04,126 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:04,126 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/40de5dff305b4cba92d1c8052a39e024, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/757de47b84d44bf692b9238cd4fce127, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6253a463c3bb44cdb613fb92ecc165aa, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/2244f704dfef49e18474f580d5367215] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=47.8 K 2024-11-14T07:16:04,127 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 40de5dff305b4cba92d1c8052a39e024, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568560028 2024-11-14T07:16:04,127 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting d486d2d9d59f4b41937cb9c4caef4c14, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568560028 2024-11-14T07:16:04,128 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 757de47b84d44bf692b9238cd4fce127, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731568560473 2024-11-14T07:16:04,128 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d775e00fb9541a1b7d86ca7a98955ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731568560473 2024-11-14T07:16:04,129 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6253a463c3bb44cdb613fb92ecc165aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731568561087 2024-11-14T07:16:04,129 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebab44eeee5c4637ba17715af1c99c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731568561087 2024-11-14T07:16:04,130 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eb5933fd17a4e61b312e5f1bfa272e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731568562763 2024-11-14T07:16:04,130 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2244f704dfef49e18474f580d5367215, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731568562763 2024-11-14T07:16:04,149 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:04,150 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/9af6071cfe0e4a469cfe579d317b6b32 is 50, key is test_row_0/B:col10/1731568563933/Put/seqid=0 2024-11-14T07:16:04,163 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#34 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:04,164 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/a2a42ccb823e4d4ca463f8d0c8781df0 is 50, key is test_row_0/A:col10/1731568563933/Put/seqid=0 2024-11-14T07:16:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741871_1047 (size=12677) 2024-11-14T07:16:04,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741870_1046 (size=12677) 2024-11-14T07:16:04,221 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/9af6071cfe0e4a469cfe579d317b6b32 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9af6071cfe0e4a469cfe579d317b6b32 2024-11-14T07:16:04,250 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into 9af6071cfe0e4a469cfe579d317b6b32(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:04,250 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:04,250 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=12, startTime=1731568564122; duration=0sec 2024-11-14T07:16:04,251 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:04,251 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:04,251 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:04,258 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:04,258 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:04,258 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:04,259 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/99b9f2b02a264a2aaca3cf6ee4c33bb5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d270aa32447b4c2e823d4f257a3e89a5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a3424c78206549a0a288dbd3d9ba94a5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/24552f08d61c478ba1f9150719cb773c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=47.8 K 2024-11-14T07:16:04,259 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b9f2b02a264a2aaca3cf6ee4c33bb5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568560028 2024-11-14T07:16:04,260 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d270aa32447b4c2e823d4f257a3e89a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731568560473 2024-11-14T07:16:04,261 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a3424c78206549a0a288dbd3d9ba94a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731568561087 2024-11-14T07:16:04,262 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 24552f08d61c478ba1f9150719cb773c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731568562763 2024-11-14T07:16:04,281 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#35 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:04,282 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/1b5cbcb9118940ea82ec1e3e3f27c31b is 50, key is test_row_0/C:col10/1731568563933/Put/seqid=0 2024-11-14T07:16:04,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741872_1048 (size=12677) 2024-11-14T07:16:04,307 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/1b5cbcb9118940ea82ec1e3e3f27c31b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/1b5cbcb9118940ea82ec1e3e3f27c31b 2024-11-14T07:16:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:04,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:04,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/dc4721307f4c47d8b64456829a60bb6a is 50, key is test_row_0/A:col10/1731568564306/Put/seqid=0 2024-11-14T07:16:04,333 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into 1b5cbcb9118940ea82ec1e3e3f27c31b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:04,333 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:04,333 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=12, startTime=1731568564123; duration=0sec 2024-11-14T07:16:04,334 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:04,334 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:04,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741873_1049 (size=12301) 2024-11-14T07:16:04,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/dc4721307f4c47d8b64456829a60bb6a 2024-11-14T07:16:04,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/fe60ab50e39449cd9827237b5743ebef is 50, key is test_row_0/B:col10/1731568564306/Put/seqid=0 2024-11-14T07:16:04,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741874_1050 (size=12301) 2024-11-14T07:16:04,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568564350, max timestamp=1731568564395Num uniques 859; 2024-11-14T07:16:04,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568564350, max timestamp=1731568564395Num uniques 859; 2024-11-14T07:16:04,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568564350, max timestamp=1731568564395Num uniques 859; 2024-11-14T07:16:04,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:04,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:04,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568624460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568624462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568624582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568624582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,609 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/a2a42ccb823e4d4ca463f8d0c8781df0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/a2a42ccb823e4d4ca463f8d0c8781df0 2024-11-14T07:16:04,621 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into a2a42ccb823e4d4ca463f8d0c8781df0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:04,621 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:04,621 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=12, startTime=1731568564122; duration=0sec 2024-11-14T07:16:04,621 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:04,621 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:16:04,702 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:04,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T07:16:04,705 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:16:04,706 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:04,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:04,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/fe60ab50e39449cd9827237b5743ebef 2024-11-14T07:16:04,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568624792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568624792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:04,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/456fcbf813b84dd3988cabe957510623 is 50, key is test_row_0/C:col10/1731568564306/Put/seqid=0 2024-11-14T07:16:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:16:04,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741875_1051 (size=12301) 2024-11-14T07:16:04,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/456fcbf813b84dd3988cabe957510623 2024-11-14T07:16:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/dc4721307f4c47d8b64456829a60bb6a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/dc4721307f4c47d8b64456829a60bb6a 2024-11-14T07:16:04,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/dc4721307f4c47d8b64456829a60bb6a, entries=150, sequenceid=281, filesize=12.0 K 2024-11-14T07:16:04,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/fe60ab50e39449cd9827237b5743ebef as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/fe60ab50e39449cd9827237b5743ebef 2024-11-14T07:16:04,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:16:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:04,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:04,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/fe60ab50e39449cd9827237b5743ebef, entries=150, sequenceid=281, filesize=12.0 K 2024-11-14T07:16:04,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/456fcbf813b84dd3988cabe957510623 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/456fcbf813b84dd3988cabe957510623 2024-11-14T07:16:04,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/456fcbf813b84dd3988cabe957510623, entries=150, sequenceid=281, filesize=12.0 K 2024-11-14T07:16:04,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 349b5cba99ebf7c21762f59b06a8ec03 in 584ms, sequenceid=281, compaction requested=false 2024-11-14T07:16:04,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:05,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:16:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:05,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T07:16:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:16:05,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/2ff42247a96946ab87ff8fa08f5d8cf9 is 50, key is test_row_0/A:col10/1731568564450/Put/seqid=0 2024-11-14T07:16:05,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741876_1052 (size=12301) 2024-11-14T07:16:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:05,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:05,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568625122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568625128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568625232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568625243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568625312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568625312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,316 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,316 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568625322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,323 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:05,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:16:05,439 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/2ff42247a96946ab87ff8fa08f5d8cf9 2024-11-14T07:16:05,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568625453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568625453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/5c16a914767a42ca979f55a300b13360 is 50, key is test_row_0/B:col10/1731568564450/Put/seqid=0 2024-11-14T07:16:05,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741877_1053 (size=12301) 2024-11-14T07:16:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568625762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568625762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:16:05,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/5c16a914767a42ca979f55a300b13360 2024-11-14T07:16:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/3be77a434e6e4437ae659e89d611681a is 50, key is test_row_0/C:col10/1731568564450/Put/seqid=0 2024-11-14T07:16:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741878_1054 (size=12301) 2024-11-14T07:16:06,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568626272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568626272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:06,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/3be77a434e6e4437ae659e89d611681a 2024-11-14T07:16:06,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/2ff42247a96946ab87ff8fa08f5d8cf9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2ff42247a96946ab87ff8fa08f5d8cf9 2024-11-14T07:16:06,315 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2ff42247a96946ab87ff8fa08f5d8cf9, entries=150, sequenceid=319, filesize=12.0 K 2024-11-14T07:16:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/5c16a914767a42ca979f55a300b13360 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5c16a914767a42ca979f55a300b13360 2024-11-14T07:16:06,328 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5c16a914767a42ca979f55a300b13360, entries=150, sequenceid=319, filesize=12.0 K 2024-11-14T07:16:06,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/3be77a434e6e4437ae659e89d611681a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3be77a434e6e4437ae659e89d611681a 2024-11-14T07:16:06,336 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3be77a434e6e4437ae659e89d611681a, entries=150, sequenceid=319, filesize=12.0 K 2024-11-14T07:16:06,337 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 349b5cba99ebf7c21762f59b06a8ec03 in 1316ms, sequenceid=319, compaction requested=true 2024-11-14T07:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T07:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T07:16:06,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T07:16:06,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-11-14T07:16:06,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6400 sec 2024-11-14T07:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:16:06,852 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:06,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T07:16:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:16:06,856 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:06,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:06,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:16:07,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:16:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:07,010 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:07,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/13b6bd00eb1e4032a39975ccf6af18d4 is 50, key is test_row_0/A:col10/1731568565125/Put/seqid=0 2024-11-14T07:16:07,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741879_1055 (size=12301) 2024-11-14T07:16:07,029 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/13b6bd00eb1e4032a39975ccf6af18d4 2024-11-14T07:16:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/09dd8d947cb245a3b6fc6e1559ac5d24 is 50, key is test_row_0/B:col10/1731568565125/Put/seqid=0 2024-11-14T07:16:07,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741880_1056 (size=12301) 2024-11-14T07:16:07,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/09dd8d947cb245a3b6fc6e1559ac5d24 2024-11-14T07:16:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/928e88d04f58453d9d5cf245ff3e0ccb is 50, key is test_row_0/C:col10/1731568565125/Put/seqid=0 2024-11-14T07:16:07,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741881_1057 (size=12301) 2024-11-14T07:16:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:16:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:07,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:07,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568567310, max timestamp=1731568567337Num uniques 859; 2024-11-14T07:16:07,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:07,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568567310, max timestamp=1731568567337Num uniques 859; 2024-11-14T07:16:07,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:07,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568567310, max timestamp=1731568567337Num uniques 859; 2024-11-14T07:16:07,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568627415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568627415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:16:07,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/928e88d04f58453d9d5cf245ff3e0ccb 2024-11-14T07:16:07,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/13b6bd00eb1e4032a39975ccf6af18d4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/13b6bd00eb1e4032a39975ccf6af18d4 2024-11-14T07:16:07,503 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/13b6bd00eb1e4032a39975ccf6af18d4, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T07:16:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/09dd8d947cb245a3b6fc6e1559ac5d24 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/09dd8d947cb245a3b6fc6e1559ac5d24 2024-11-14T07:16:07,516 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/09dd8d947cb245a3b6fc6e1559ac5d24, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T07:16:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/928e88d04f58453d9d5cf245ff3e0ccb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/928e88d04f58453d9d5cf245ff3e0ccb 2024-11-14T07:16:07,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/928e88d04f58453d9d5cf245ff3e0ccb, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T07:16:07,531 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 349b5cba99ebf7c21762f59b06a8ec03 in 521ms, sequenceid=329, compaction requested=true 2024-11-14T07:16:07,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:07,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:07,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T07:16:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T07:16:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:07,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-14T07:16:07,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:07,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:07,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:07,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:07,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:07,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:07,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T07:16:07,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 675 msec 2024-11-14T07:16:07,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 689 msec 2024-11-14T07:16:07,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ee56767e5de1403492ad3892fce06e9e is 50, key is test_row_0/A:col10/1731568567392/Put/seqid=0 2024-11-14T07:16:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568627556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568627560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741882_1058 (size=14741) 2024-11-14T07:16:07,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ee56767e5de1403492ad3892fce06e9e 2024-11-14T07:16:07,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/abaaae37931646b4bd8acda37bd9b314 is 50, key is test_row_0/B:col10/1731568567392/Put/seqid=0 2024-11-14T07:16:07,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741883_1059 (size=12301) 2024-11-14T07:16:07,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568627662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568627672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568627872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568627882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:16:07,992 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:07,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T07:16:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:16:07,994 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:07,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/abaaae37931646b4bd8acda37bd9b314 2024-11-14T07:16:07,995 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:07,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:08,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/fcc093bb9791494d900e907e8697c364 is 50, key is test_row_0/C:col10/1731568567392/Put/seqid=0 2024-11-14T07:16:08,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741884_1060 (size=12301) 2024-11-14T07:16:08,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/fcc093bb9791494d900e907e8697c364 2024-11-14T07:16:08,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ee56767e5de1403492ad3892fce06e9e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ee56767e5de1403492ad3892fce06e9e 2024-11-14T07:16:08,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ee56767e5de1403492ad3892fce06e9e, entries=200, sequenceid=366, filesize=14.4 K 2024-11-14T07:16:08,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/abaaae37931646b4bd8acda37bd9b314 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/abaaae37931646b4bd8acda37bd9b314 2024-11-14T07:16:08,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/abaaae37931646b4bd8acda37bd9b314, entries=150, sequenceid=366, filesize=12.0 K 2024-11-14T07:16:08,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/fcc093bb9791494d900e907e8697c364 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fcc093bb9791494d900e907e8697c364 2024-11-14T07:16:08,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fcc093bb9791494d900e907e8697c364, entries=150, sequenceid=366, filesize=12.0 K 2024-11-14T07:16:08,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 349b5cba99ebf7c21762f59b06a8ec03 in 513ms, sequenceid=366, compaction requested=true 2024-11-14T07:16:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:08,050 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:16:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:08,050 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:16:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:08,052 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61881 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:16:08,052 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:08,052 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:08,053 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9af6071cfe0e4a469cfe579d317b6b32, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/fe60ab50e39449cd9827237b5743ebef, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5c16a914767a42ca979f55a300b13360, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/09dd8d947cb245a3b6fc6e1559ac5d24, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/abaaae37931646b4bd8acda37bd9b314] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=60.4 K 2024-11-14T07:16:08,056 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64321 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:16:08,056 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af6071cfe0e4a469cfe579d317b6b32, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731568562763 2024-11-14T07:16:08,056 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:08,056 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:08,056 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/a2a42ccb823e4d4ca463f8d0c8781df0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/dc4721307f4c47d8b64456829a60bb6a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2ff42247a96946ab87ff8fa08f5d8cf9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/13b6bd00eb1e4032a39975ccf6af18d4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ee56767e5de1403492ad3892fce06e9e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=62.8 K 2024-11-14T07:16:08,057 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting fe60ab50e39449cd9827237b5743ebef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731568563951 2024-11-14T07:16:08,057 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a42ccb823e4d4ca463f8d0c8781df0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731568562763 2024-11-14T07:16:08,058 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc4721307f4c47d8b64456829a60bb6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731568563951 2024-11-14T07:16:08,058 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c16a914767a42ca979f55a300b13360, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731568564450 2024-11-14T07:16:08,058 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 09dd8d947cb245a3b6fc6e1559ac5d24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731568565117 2024-11-14T07:16:08,058 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff42247a96946ab87ff8fa08f5d8cf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731568564450 2024-11-14T07:16:08,059 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b6bd00eb1e4032a39975ccf6af18d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731568565117 2024-11-14T07:16:08,059 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting abaaae37931646b4bd8acda37bd9b314, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731568567392 2024-11-14T07:16:08,059 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee56767e5de1403492ad3892fce06e9e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731568567392 2024-11-14T07:16:08,073 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:08,074 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ed11af2bad9f4cd6a95e3401f7119908 is 50, key is test_row_0/A:col10/1731568567392/Put/seqid=0 2024-11-14T07:16:08,077 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:08,077 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/8956ee23fe6a4e198d5449a27a2d29b6 is 50, key is test_row_0/B:col10/1731568567392/Put/seqid=0 2024-11-14T07:16:08,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741885_1061 (size=12847) 2024-11-14T07:16:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:16:08,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741886_1062 (size=12847) 2024-11-14T07:16:08,115 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/8956ee23fe6a4e198d5449a27a2d29b6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/8956ee23fe6a4e198d5449a27a2d29b6 2024-11-14T07:16:08,124 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into 8956ee23fe6a4e198d5449a27a2d29b6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:08,124 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:08,124 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=11, startTime=1731568568050; duration=0sec 2024-11-14T07:16:08,125 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:08,125 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:08,125 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:16:08,128 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61881 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:16:08,128 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:08,128 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:08,128 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/1b5cbcb9118940ea82ec1e3e3f27c31b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/456fcbf813b84dd3988cabe957510623, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3be77a434e6e4437ae659e89d611681a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/928e88d04f58453d9d5cf245ff3e0ccb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fcc093bb9791494d900e907e8697c364] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=60.4 K 2024-11-14T07:16:08,129 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b5cbcb9118940ea82ec1e3e3f27c31b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731568562763 2024-11-14T07:16:08,129 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 456fcbf813b84dd3988cabe957510623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731568563951 2024-11-14T07:16:08,129 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 3be77a434e6e4437ae659e89d611681a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731568564450 2024-11-14T07:16:08,130 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 928e88d04f58453d9d5cf245ff3e0ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731568565117 2024-11-14T07:16:08,130 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc093bb9791494d900e907e8697c364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731568567392 2024-11-14T07:16:08,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T07:16:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:08,148 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T07:16:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:08,150 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#50 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:08,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/eebc316b35ef4a618b599f462a24e806 is 50, key is test_row_0/A:col10/1731568567558/Put/seqid=0 2024-11-14T07:16:08,155 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/fa28ecf915b448dfb8df8bf7fad184df is 50, key is test_row_0/C:col10/1731568567392/Put/seqid=0 2024-11-14T07:16:08,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741888_1064 (size=12847) 2024-11-14T07:16:08,173 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/fa28ecf915b448dfb8df8bf7fad184df as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fa28ecf915b448dfb8df8bf7fad184df 2024-11-14T07:16:08,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741887_1063 (size=9857) 2024-11-14T07:16:08,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/eebc316b35ef4a618b599f462a24e806 2024-11-14T07:16:08,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:08,192 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into fa28ecf915b448dfb8df8bf7fad184df(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:08,192 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:08,192 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=11, startTime=1731568568050; duration=0sec 2024-11-14T07:16:08,192 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:08,192 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/f7a62e2957e84b438cb4e7fe05f747fa is 50, key is test_row_0/B:col10/1731568567558/Put/seqid=0 2024-11-14T07:16:08,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741889_1065 (size=9857) 2024-11-14T07:16:08,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568568213, max timestamp=1731568568231Num uniques 859; 2024-11-14T07:16:08,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568568213, max timestamp=1731568568231Num uniques 859; 2024-11-14T07:16:08,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:08,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:08,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568568213, max timestamp=1731568568231Num uniques 859; 2024-11-14T07:16:08,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568628276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:08,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568628277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:16:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568628392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:08,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568628392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:08,498 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ed11af2bad9f4cd6a95e3401f7119908 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ed11af2bad9f4cd6a95e3401f7119908 2024-11-14T07:16:08,506 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into ed11af2bad9f4cd6a95e3401f7119908(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:08,507 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:08,507 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=11, startTime=1731568568050; duration=0sec 2024-11-14T07:16:08,507 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:08,507 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568628602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568628602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:08,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/f7a62e2957e84b438cb4e7fe05f747fa 2024-11-14T07:16:08,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/de8a281b9316466db83d20c4922a599f is 50, key is test_row_0/C:col10/1731568567558/Put/seqid=0 2024-11-14T07:16:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:16:08,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741890_1066 (size=9857) 2024-11-14T07:16:08,632 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/de8a281b9316466db83d20c4922a599f 2024-11-14T07:16:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/eebc316b35ef4a618b599f462a24e806 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/eebc316b35ef4a618b599f462a24e806 2024-11-14T07:16:08,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/eebc316b35ef4a618b599f462a24e806, entries=100, sequenceid=376, filesize=9.6 K 2024-11-14T07:16:08,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/f7a62e2957e84b438cb4e7fe05f747fa as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/f7a62e2957e84b438cb4e7fe05f747fa 2024-11-14T07:16:08,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/f7a62e2957e84b438cb4e7fe05f747fa, entries=100, sequenceid=376, filesize=9.6 K 2024-11-14T07:16:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/de8a281b9316466db83d20c4922a599f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/de8a281b9316466db83d20c4922a599f 2024-11-14T07:16:08,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/de8a281b9316466db83d20c4922a599f, entries=100, sequenceid=376, filesize=9.6 K 2024-11-14T07:16:08,670 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 349b5cba99ebf7c21762f59b06a8ec03 in 522ms, sequenceid=376, compaction requested=false 2024-11-14T07:16:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T07:16:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T07:16:08,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T07:16:08,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 677 msec 2024-11-14T07:16:08,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 684 msec 2024-11-14T07:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:08,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=147.60 KB heapSize=415.92 KB 2024-11-14T07:16:08,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568568249, max timestamp=1731568568913Num uniques 1118; 2024-11-14T07:16:08,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-14T07:16:08,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568568249, max timestamp=1731568568913Num uniques 1118; 2024-11-14T07:16:08,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-14T07:16:08,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568568249, max timestamp=1731568568913Num uniques 1118; 2024-11-14T07:16:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:08,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-14T07:16:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:08,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/e6b9340d2c634c2389415cd352840463 is 50, key is test_row_0/A:col10/1731568568913/Put/seqid=0 2024-11-14T07:16:08,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741891_1067 (size=17181) 2024-11-14T07:16:08,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731568568954, max timestamp=1731568568971Num uniques 859; 2024-11-14T07:16:08,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731568568954, max timestamp=1731568568971Num uniques 859; 2024-11-14T07:16:08,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T07:16:08,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T07:16:08,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731568568954, max timestamp=1731568568971Num uniques 859; 2024-11-14T07:16:08,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T07:16:09,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568629016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568629018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:16:09,132 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:09,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T07:16:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:16:09,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:09,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:09,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:09,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568629131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568629132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:16:09,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:16:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43214 deadline: 1731568629332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,334 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568629342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568629343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43182 deadline: 1731568629351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,353 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43208 deadline: 1731568629352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,353 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 349b5cba99ebf7c21762f59b06a8ec03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/e6b9340d2c634c2389415cd352840463 2024-11-14T07:16:09,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/77838e47f38449918fd52d9c12273000 is 50, key is test_row_0/B:col10/1731568568913/Put/seqid=0 2024-11-14T07:16:09,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741892_1068 (size=12301) 2024-11-14T07:16:09,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:16:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:16:09,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:16:09,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:09,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568629652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568629652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:09,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:16:09,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:09,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:16:09,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/77838e47f38449918fd52d9c12273000 2024-11-14T07:16:09,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a96b31b4498b4a2d8a1bc29fd42469d9 is 50, key is test_row_0/C:col10/1731568568913/Put/seqid=0 2024-11-14T07:16:09,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741893_1069 (size=12301) 2024-11-14T07:16:09,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a96b31b4498b4a2d8a1bc29fd42469d9 2024-11-14T07:16:09,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/e6b9340d2c634c2389415cd352840463 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e6b9340d2c634c2389415cd352840463 2024-11-14T07:16:09,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e6b9340d2c634c2389415cd352840463, entries=250, sequenceid=416, filesize=16.8 K 2024-11-14T07:16:09,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/77838e47f38449918fd52d9c12273000 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/77838e47f38449918fd52d9c12273000 2024-11-14T07:16:09,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/77838e47f38449918fd52d9c12273000, entries=150, sequenceid=416, filesize=12.0 K 2024-11-14T07:16:09,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a96b31b4498b4a2d8a1bc29fd42469d9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a96b31b4498b4a2d8a1bc29fd42469d9 2024-11-14T07:16:09,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a96b31b4498b4a2d8a1bc29fd42469d9, entries=150, sequenceid=416, filesize=12.0 K 2024-11-14T07:16:09,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=161.02 KB/164880 for 349b5cba99ebf7c21762f59b06a8ec03 in 931ms, sequenceid=416, compaction requested=true 2024-11-14T07:16:09,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:09,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:09,846 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:09,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:09,846 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:09,851 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39885 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:09,852 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:09,852 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,852 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:09,852 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ed11af2bad9f4cd6a95e3401f7119908, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/eebc316b35ef4a618b599f462a24e806, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e6b9340d2c634c2389415cd352840463] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=39.0 K 2024-11-14T07:16:09,852 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:09,852 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,852 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/8956ee23fe6a4e198d5449a27a2d29b6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/f7a62e2957e84b438cb4e7fe05f747fa, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/77838e47f38449918fd52d9c12273000] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=34.2 K 2024-11-14T07:16:09,853 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed11af2bad9f4cd6a95e3401f7119908, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731568567392 2024-11-14T07:16:09,853 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8956ee23fe6a4e198d5449a27a2d29b6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731568567392 2024-11-14T07:16:09,854 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting eebc316b35ef4a618b599f462a24e806, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731568567539 2024-11-14T07:16:09,855 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a62e2957e84b438cb4e7fe05f747fa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731568567539 2024-11-14T07:16:09,855 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b9340d2c634c2389415cd352840463, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731568568249 2024-11-14T07:16:09,855 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 77838e47f38449918fd52d9c12273000, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731568568249 2024-11-14T07:16:09,879 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:09,879 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:09,879 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/6cebf50fa5184153b135ff3833c2f5fb is 50, key is test_row_0/A:col10/1731568568913/Put/seqid=0 2024-11-14T07:16:09,879 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/63e5438817da4fcd82d0d9e4400109f6 is 50, key is test_row_0/B:col10/1731568568913/Put/seqid=0 2024-11-14T07:16:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741895_1071 (size=12949) 2024-11-14T07:16:09,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741894_1070 (size=12949) 2024-11-14T07:16:09,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:16:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,907 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=161.02 KB heapSize=445.92 KB 2024-11-14T07:16:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:09,917 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/6cebf50fa5184153b135ff3833c2f5fb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6cebf50fa5184153b135ff3833c2f5fb 2024-11-14T07:16:09,918 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/63e5438817da4fcd82d0d9e4400109f6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63e5438817da4fcd82d0d9e4400109f6 2024-11-14T07:16:09,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/40489e4da6814941bbb6ca8b7fd4e258 is 50, key is test_row_0/A:col10/1731568569016/Put/seqid=0 2024-11-14T07:16:09,927 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into 6cebf50fa5184153b135ff3833c2f5fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:09,927 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:09,927 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=13, startTime=1731568569846; duration=0sec 2024-11-14T07:16:09,928 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:09,928 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:09,928 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:09,928 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into 63e5438817da4fcd82d0d9e4400109f6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:09,928 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:09,929 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=13, startTime=1731568569846; duration=0sec 2024-11-14T07:16:09,929 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:09,929 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:09,933 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:09,933 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:09,933 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:09,933 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fa28ecf915b448dfb8df8bf7fad184df, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/de8a281b9316466db83d20c4922a599f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a96b31b4498b4a2d8a1bc29fd42469d9] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=34.2 K 2024-11-14T07:16:09,934 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa28ecf915b448dfb8df8bf7fad184df, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731568567392 2024-11-14T07:16:09,934 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting de8a281b9316466db83d20c4922a599f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731568567539 2024-11-14T07:16:09,934 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96b31b4498b4a2d8a1bc29fd42469d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731568568249 2024-11-14T07:16:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741896_1072 (size=12301) 2024-11-14T07:16:09,947 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:09,948 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d8a05ea4aa6b40d3b91eeff20126b730 is 50, key is test_row_0/C:col10/1731568568913/Put/seqid=0 2024-11-14T07:16:09,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741897_1073 (size=12949) 2024-11-14T07:16:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:10,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568630173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568630181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:16:10,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568630293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568630293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,348 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/40489e4da6814941bbb6ca8b7fd4e258 2024-11-14T07:16:10,367 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d8a05ea4aa6b40d3b91eeff20126b730 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8a05ea4aa6b40d3b91eeff20126b730 2024-11-14T07:16:10,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/27ee877c976245d18a71f3d040c9a209 is 50, key is test_row_0/B:col10/1731568569016/Put/seqid=0 2024-11-14T07:16:10,396 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into d8a05ea4aa6b40d3b91eeff20126b730(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:10,396 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:10,396 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=13, startTime=1731568569847; duration=0sec 2024-11-14T07:16:10,396 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:10,396 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:10,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741898_1074 (size=12301) 2024-11-14T07:16:10,425 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/27ee877c976245d18a71f3d040c9a209 2024-11-14T07:16:10,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d8b46ce0921e4748b616f945f893091a is 50, key is test_row_0/C:col10/1731568569016/Put/seqid=0 2024-11-14T07:16:10,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741899_1075 (size=12301) 2024-11-14T07:16:10,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568630502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568630512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568630833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568630833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:10,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d8b46ce0921e4748b616f945f893091a 2024-11-14T07:16:10,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/40489e4da6814941bbb6ca8b7fd4e258 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/40489e4da6814941bbb6ca8b7fd4e258 2024-11-14T07:16:10,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/40489e4da6814941bbb6ca8b7fd4e258, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T07:16:10,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/27ee877c976245d18a71f3d040c9a209 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/27ee877c976245d18a71f3d040c9a209 2024-11-14T07:16:10,944 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/27ee877c976245d18a71f3d040c9a209, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T07:16:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/d8b46ce0921e4748b616f945f893091a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8b46ce0921e4748b616f945f893091a 2024-11-14T07:16:10,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8b46ce0921e4748b616f945f893091a, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T07:16:10,970 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=26.84 KB/27480 for 349b5cba99ebf7c21762f59b06a8ec03 in 1059ms, sequenceid=453, compaction requested=false 2024-11-14T07:16:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T07:16:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T07:16:10,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T07:16:10,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8440 sec 2024-11-14T07:16:10,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.8520 sec 2024-11-14T07:16:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:16:11,282 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:11,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T07:16:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:16:11,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:11,291 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:11,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:11,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:11,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/f6a59db667ea449298de999d4faacca2 is 50, key is test_row_0/A:col10/1731568571348/Put/seqid=0 2024-11-14T07:16:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:16:11,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741900_1076 (size=14741) 2024-11-14T07:16:11,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/f6a59db667ea449298de999d4faacca2 2024-11-14T07:16:11,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/db17bbcdd7ef469681ef424d2c77881c is 50, key is test_row_0/B:col10/1731568571348/Put/seqid=0 2024-11-14T07:16:11,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568571401, max timestamp=1731568571440Num uniques 859; 2024-11-14T07:16:11,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568571401, max timestamp=1731568571440Num uniques 859; 2024-11-14T07:16:11,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:11,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:11,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:16:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:11,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568571412, max timestamp=1731568571440Num uniques 859; 2024-11-14T07:16:11,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:11,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:16:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:11,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741901_1077 (size=12301) 2024-11-14T07:16:11,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/db17bbcdd7ef469681ef424d2c77881c 2024-11-14T07:16:11,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/92e4c716eba14d1c8a3a6d35c6ba5e1b is 50, key is test_row_0/C:col10/1731568571348/Put/seqid=0 2024-11-14T07:16:11,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741902_1078 (size=12301) 2024-11-14T07:16:11,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/92e4c716eba14d1c8a3a6d35c6ba5e1b 2024-11-14T07:16:11,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/f6a59db667ea449298de999d4faacca2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f6a59db667ea449298de999d4faacca2 2024-11-14T07:16:11,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f6a59db667ea449298de999d4faacca2, entries=200, sequenceid=467, filesize=14.4 K 2024-11-14T07:16:11,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/db17bbcdd7ef469681ef424d2c77881c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/db17bbcdd7ef469681ef424d2c77881c 2024-11-14T07:16:11,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/db17bbcdd7ef469681ef424d2c77881c, entries=150, sequenceid=467, filesize=12.0 K 2024-11-14T07:16:11,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/92e4c716eba14d1c8a3a6d35c6ba5e1b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/92e4c716eba14d1c8a3a6d35c6ba5e1b 2024-11-14T07:16:11,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568631566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568631572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/92e4c716eba14d1c8a3a6d35c6ba5e1b, entries=150, sequenceid=467, filesize=12.0 K 2024-11-14T07:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:11,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 349b5cba99ebf7c21762f59b06a8ec03 in 232ms, sequenceid=467, compaction requested=true 2024-11-14T07:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:11,582 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:11,582 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:11,583 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:11,583 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:11,583 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:11,583 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:11,583 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:11,584 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:11,584 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6cebf50fa5184153b135ff3833c2f5fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/40489e4da6814941bbb6ca8b7fd4e258, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f6a59db667ea449298de999d4faacca2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=39.1 K 2024-11-14T07:16:11,584 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63e5438817da4fcd82d0d9e4400109f6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/27ee877c976245d18a71f3d040c9a209, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/db17bbcdd7ef469681ef424d2c77881c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=36.7 K 2024-11-14T07:16:11,584 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 63e5438817da4fcd82d0d9e4400109f6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731568568249 2024-11-14T07:16:11,585 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 27ee877c976245d18a71f3d040c9a209, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731568569006 2024-11-14T07:16:11,585 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting db17bbcdd7ef469681ef424d2c77881c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731568571343 2024-11-14T07:16:11,585 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cebf50fa5184153b135ff3833c2f5fb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731568568249 2024-11-14T07:16:11,586 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40489e4da6814941bbb6ca8b7fd4e258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731568569006 2024-11-14T07:16:11,586 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6a59db667ea449298de999d4faacca2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731568570179 2024-11-14T07:16:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:16:11,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:16:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:11,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-14T07:16:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:11,608 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:11,608 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/847ded74d6b54a3cad0a27b85edfb67f is 50, key is test_row_0/B:col10/1731568571348/Put/seqid=0 2024-11-14T07:16:11,611 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:11,611 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/b358825caf2e40d097f611cf1bdfc4eb is 50, key is test_row_0/A:col10/1731568571348/Put/seqid=0 2024-11-14T07:16:11,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ac2c818ce0bf49e18b043e3cf8baaade is 50, key is test_row_0/A:col10/1731568571563/Put/seqid=0 2024-11-14T07:16:11,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741903_1079 (size=13051) 2024-11-14T07:16:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:11,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741905_1081 (size=13051) 2024-11-14T07:16:11,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568631728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:11,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741904_1080 (size=12301) 2024-11-14T07:16:11,740 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ac2c818ce0bf49e18b043e3cf8baaade 2024-11-14T07:16:11,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568631730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:11,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/9fae37defb9e488cbc796862f070add8 is 50, key is test_row_0/B:col10/1731568571563/Put/seqid=0 2024-11-14T07:16:11,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741906_1082 (size=12301) 2024-11-14T07:16:11,825 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/9fae37defb9e488cbc796862f070add8 2024-11-14T07:16:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/cca0e12a3d254b62b26804d465f8cfd6 is 50, key is test_row_0/C:col10/1731568571563/Put/seqid=0 2024-11-14T07:16:11,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568631842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:11,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:11,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741907_1083 (size=12301) 2024-11-14T07:16:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568631853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:16:12,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568632062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568632092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,114 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/847ded74d6b54a3cad0a27b85edfb67f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/847ded74d6b54a3cad0a27b85edfb67f 2024-11-14T07:16:12,124 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/b358825caf2e40d097f611cf1bdfc4eb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b358825caf2e40d097f611cf1bdfc4eb 2024-11-14T07:16:12,125 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into 847ded74d6b54a3cad0a27b85edfb67f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:12,125 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:12,125 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=13, startTime=1731568571582; duration=0sec 2024-11-14T07:16:12,126 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:12,126 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:12,126 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:12,128 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:12,128 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:12,128 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,129 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8a05ea4aa6b40d3b91eeff20126b730, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8b46ce0921e4748b616f945f893091a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/92e4c716eba14d1c8a3a6d35c6ba5e1b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=36.7 K 2024-11-14T07:16:12,129 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a05ea4aa6b40d3b91eeff20126b730, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731568568249 2024-11-14T07:16:12,130 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d8b46ce0921e4748b616f945f893091a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731568569006 2024-11-14T07:16:12,130 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e4c716eba14d1c8a3a6d35c6ba5e1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731568571343 2024-11-14T07:16:12,137 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into b358825caf2e40d097f611cf1bdfc4eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:12,137 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:12,137 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=13, startTime=1731568571582; duration=0sec 2024-11-14T07:16:12,137 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:12,137 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:12,162 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:12,162 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/17a70d21ba08467e93469f507e1dda7b is 50, key is test_row_0/C:col10/1731568571348/Put/seqid=0 2024-11-14T07:16:12,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741908_1084 (size=13051) 2024-11-14T07:16:12,269 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/cca0e12a3d254b62b26804d465f8cfd6 2024-11-14T07:16:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/ac2c818ce0bf49e18b043e3cf8baaade as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ac2c818ce0bf49e18b043e3cf8baaade 2024-11-14T07:16:12,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ac2c818ce0bf49e18b043e3cf8baaade, entries=150, sequenceid=502, filesize=12.0 K 2024-11-14T07:16:12,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/9fae37defb9e488cbc796862f070add8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9fae37defb9e488cbc796862f070add8 2024-11-14T07:16:12,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9fae37defb9e488cbc796862f070add8, entries=150, sequenceid=502, filesize=12.0 K 2024-11-14T07:16:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/cca0e12a3d254b62b26804d465f8cfd6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cca0e12a3d254b62b26804d465f8cfd6 2024-11-14T07:16:12,303 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cca0e12a3d254b62b26804d465f8cfd6, entries=150, sequenceid=502, filesize=12.0 K 2024-11-14T07:16:12,304 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for 349b5cba99ebf7c21762f59b06a8ec03 in 701ms, sequenceid=502, compaction requested=false 2024-11-14T07:16:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T07:16:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T07:16:12,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T07:16:12,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-14T07:16:12,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0260 sec 2024-11-14T07:16:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:12,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:16:12,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:12,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:12,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:12,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:12,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:12,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/343b728df1664be59401d158455ca3ca is 50, key is test_row_0/A:col10/1731568572375/Put/seqid=0 2024-11-14T07:16:12,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741909_1085 (size=14741) 2024-11-14T07:16:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:16:12,422 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:12,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-14T07:16:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:16:12,425 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:12,426 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:12,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568632433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568632434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:16:12,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568632542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568632551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T07:16:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:12,599 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/17a70d21ba08467e93469f507e1dda7b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/17a70d21ba08467e93469f507e1dda7b 2024-11-14T07:16:12,606 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into 17a70d21ba08467e93469f507e1dda7b(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:12,606 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:12,607 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=13, startTime=1731568571582; duration=0sec 2024-11-14T07:16:12,607 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:12,607 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:12,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T07:16:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:16:12,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568632752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568632761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:12,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/343b728df1664be59401d158455ca3ca 2024-11-14T07:16:12,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/d6612505267140ff81c0c0d3ca5942a5 is 50, key is test_row_0/B:col10/1731568572375/Put/seqid=0 2024-11-14T07:16:12,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741910_1086 (size=12301) 2024-11-14T07:16:12,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/d6612505267140ff81c0c0d3ca5942a5 2024-11-14T07:16:12,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a84bfb73040e410683eb4cf92b231bbb is 50, key is test_row_0/C:col10/1731568572375/Put/seqid=0 2024-11-14T07:16:12,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741911_1087 (size=12301) 2024-11-14T07:16:12,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a84bfb73040e410683eb4cf92b231bbb 2024-11-14T07:16:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/343b728df1664be59401d158455ca3ca as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/343b728df1664be59401d158455ca3ca 2024-11-14T07:16:12,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/343b728df1664be59401d158455ca3ca, entries=200, sequenceid=516, filesize=14.4 K 2024-11-14T07:16:12,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/d6612505267140ff81c0c0d3ca5942a5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/d6612505267140ff81c0c0d3ca5942a5 2024-11-14T07:16:12,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/d6612505267140ff81c0c0d3ca5942a5, entries=150, sequenceid=516, filesize=12.0 K 2024-11-14T07:16:12,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/a84bfb73040e410683eb4cf92b231bbb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a84bfb73040e410683eb4cf92b231bbb 2024-11-14T07:16:12,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a84bfb73040e410683eb4cf92b231bbb, entries=150, sequenceid=516, filesize=12.0 K 2024-11-14T07:16:12,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 349b5cba99ebf7c21762f59b06a8ec03 in 470ms, sequenceid=516, compaction requested=true 2024-11-14T07:16:12,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:12,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:12,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:12,847 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:12,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:12,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:12,847 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:12,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:12,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:12,848 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:12,848 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:12,848 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:12,848 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:12,848 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,848 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,849 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b358825caf2e40d097f611cf1bdfc4eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ac2c818ce0bf49e18b043e3cf8baaade, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/343b728df1664be59401d158455ca3ca] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=39.2 K 2024-11-14T07:16:12,849 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/847ded74d6b54a3cad0a27b85edfb67f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9fae37defb9e488cbc796862f070add8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/d6612505267140ff81c0c0d3ca5942a5] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=36.8 K 2024-11-14T07:16:12,849 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 847ded74d6b54a3cad0a27b85edfb67f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731568571343 2024-11-14T07:16:12,849 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting b358825caf2e40d097f611cf1bdfc4eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731568571343 2024-11-14T07:16:12,849 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fae37defb9e488cbc796862f070add8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731568571528 2024-11-14T07:16:12,849 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac2c818ce0bf49e18b043e3cf8baaade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731568571528 2024-11-14T07:16:12,850 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343b728df1664be59401d158455ca3ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731568571698 2024-11-14T07:16:12,850 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d6612505267140ff81c0c0d3ca5942a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731568571698 2024-11-14T07:16:12,858 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:12,858 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/f2866b21f8e04b49bb00257844dedfd3 is 50, key is test_row_0/A:col10/1731568572375/Put/seqid=0 2024-11-14T07:16:12,859 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:12,859 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/10e81d528c2c4c38987363e6498cc056 is 50, key is test_row_0/B:col10/1731568572375/Put/seqid=0 2024-11-14T07:16:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741913_1089 (size=13153) 2024-11-14T07:16:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741912_1088 (size=13153) 2024-11-14T07:16:12,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T07:16:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:12,889 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:16:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:12,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:12,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/09dd5194bebb49cc829b8c48810b0cfb is 50, key is test_row_0/A:col10/1731568572410/Put/seqid=0 2024-11-14T07:16:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741914_1090 (size=12301) 2024-11-14T07:16:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:16:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:13,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568633087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568633091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568633202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568633202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,271 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/10e81d528c2c4c38987363e6498cc056 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/10e81d528c2c4c38987363e6498cc056 2024-11-14T07:16:13,275 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/f2866b21f8e04b49bb00257844dedfd3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f2866b21f8e04b49bb00257844dedfd3 2024-11-14T07:16:13,279 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into 10e81d528c2c4c38987363e6498cc056(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:13,279 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:13,279 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=13, startTime=1731568572847; duration=0sec 2024-11-14T07:16:13,279 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:13,279 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:13,279 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:13,281 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:13,281 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:13,281 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:13,281 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/17a70d21ba08467e93469f507e1dda7b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cca0e12a3d254b62b26804d465f8cfd6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a84bfb73040e410683eb4cf92b231bbb] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=36.8 K 2024-11-14T07:16:13,282 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a70d21ba08467e93469f507e1dda7b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731568571343 2024-11-14T07:16:13,283 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cca0e12a3d254b62b26804d465f8cfd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731568571528 2024-11-14T07:16:13,283 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a84bfb73040e410683eb4cf92b231bbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731568571698 2024-11-14T07:16:13,285 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into f2866b21f8e04b49bb00257844dedfd3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:13,285 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:13,285 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=13, startTime=1731568572846; duration=0sec 2024-11-14T07:16:13,286 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:13,286 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:13,290 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:13,291 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/7e1d3bb6ff014eee86fc7533bb69bad4 is 50, key is test_row_0/C:col10/1731568572375/Put/seqid=0 2024-11-14T07:16:13,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/09dd5194bebb49cc829b8c48810b0cfb 2024-11-14T07:16:13,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741915_1091 (size=13153) 2024-11-14T07:16:13,310 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/7e1d3bb6ff014eee86fc7533bb69bad4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/7e1d3bb6ff014eee86fc7533bb69bad4 2024-11-14T07:16:13,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/0281b0a0a3864c5e8cbf7db94f3e6e56 is 50, key is test_row_0/B:col10/1731568572410/Put/seqid=0 2024-11-14T07:16:13,317 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into 7e1d3bb6ff014eee86fc7533bb69bad4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:13,317 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:13,317 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=13, startTime=1731568572847; duration=0sec 2024-11-14T07:16:13,318 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:13,318 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:13,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741916_1092 (size=12301) 2024-11-14T07:16:13,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/0281b0a0a3864c5e8cbf7db94f3e6e56 2024-11-14T07:16:13,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/2c4c066a23c5466cb65cf611cf70b553 is 50, key is test_row_0/C:col10/1731568572410/Put/seqid=0 2024-11-14T07:16:13,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741917_1093 (size=12301) 2024-11-14T07:16:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568633412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568633412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:16:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568633722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568633722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:13,734 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/2c4c066a23c5466cb65cf611cf70b553 2024-11-14T07:16:13,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/09dd5194bebb49cc829b8c48810b0cfb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/09dd5194bebb49cc829b8c48810b0cfb 2024-11-14T07:16:13,748 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/09dd5194bebb49cc829b8c48810b0cfb, entries=150, sequenceid=537, filesize=12.0 K 2024-11-14T07:16:13,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/0281b0a0a3864c5e8cbf7db94f3e6e56 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/0281b0a0a3864c5e8cbf7db94f3e6e56 2024-11-14T07:16:13,756 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/0281b0a0a3864c5e8cbf7db94f3e6e56, entries=150, sequenceid=537, filesize=12.0 K 2024-11-14T07:16:13,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/2c4c066a23c5466cb65cf611cf70b553 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/2c4c066a23c5466cb65cf611cf70b553 2024-11-14T07:16:13,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/2c4c066a23c5466cb65cf611cf70b553, entries=150, sequenceid=537, filesize=12.0 K 2024-11-14T07:16:13,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 349b5cba99ebf7c21762f59b06a8ec03 in 874ms, sequenceid=537, compaction requested=false 2024-11-14T07:16:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-14T07:16:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-14T07:16:13,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T07:16:13,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3390 sec 2024-11-14T07:16:13,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3450 sec 2024-11-14T07:16:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:14,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:16:14,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:14,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:14,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/10cadf6b35d747a097364e7579cfc504 is 50, key is test_row_0/A:col10/1731568574234/Put/seqid=0 2024-11-14T07:16:14,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741918_1094 (size=14741) 2024-11-14T07:16:14,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568634274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568634276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:14,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568634381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:14,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568634392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:16:14,582 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:14,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-14T07:16:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:16:14,587 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:14,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:14,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:14,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568634592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568634612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:14,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/10cadf6b35d747a097364e7579cfc504 2024-11-14T07:16:14,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/ad209e2d44ab42d4bc3295c1de34764a is 50, key is test_row_0/B:col10/1731568574234/Put/seqid=0 2024-11-14T07:16:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:16:14,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741919_1095 (size=12301) 2024-11-14T07:16:14,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:16:14,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:14,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:14,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:14,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:14,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:14,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:16:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:14,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:16:14,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568634902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:14,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568634922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:15,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:16:15,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:15,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:15,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:15,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/ad209e2d44ab42d4bc3295c1de34764a 2024-11-14T07:16:15,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/461ad5ef2e5a4fa2baef81841c5b5415 is 50, key is test_row_0/C:col10/1731568574234/Put/seqid=0 2024-11-14T07:16:15,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741920_1096 (size=12301) 2024-11-14T07:16:15,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/461ad5ef2e5a4fa2baef81841c5b5415 2024-11-14T07:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/10cadf6b35d747a097364e7579cfc504 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/10cadf6b35d747a097364e7579cfc504 2024-11-14T07:16:15,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/10cadf6b35d747a097364e7579cfc504, entries=200, sequenceid=554, filesize=14.4 K 2024-11-14T07:16:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/ad209e2d44ab42d4bc3295c1de34764a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ad209e2d44ab42d4bc3295c1de34764a 2024-11-14T07:16:15,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:16:15,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ad209e2d44ab42d4bc3295c1de34764a, entries=150, sequenceid=554, filesize=12.0 K 2024-11-14T07:16:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/461ad5ef2e5a4fa2baef81841c5b5415 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/461ad5ef2e5a4fa2baef81841c5b5415 2024-11-14T07:16:15,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:15,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:15,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/461ad5ef2e5a4fa2baef81841c5b5415, entries=150, sequenceid=554, filesize=12.0 K 2024-11-14T07:16:15,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 349b5cba99ebf7c21762f59b06a8ec03 in 986ms, sequenceid=554, compaction requested=true 2024-11-14T07:16:15,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:15,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:15,221 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:15,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:15,221 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:15,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:15,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:15,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:15,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:15,223 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:15,224 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:15,224 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:15,224 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:15,224 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,224 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,224 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/10e81d528c2c4c38987363e6498cc056, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/0281b0a0a3864c5e8cbf7db94f3e6e56, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ad209e2d44ab42d4bc3295c1de34764a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=36.9 K 2024-11-14T07:16:15,224 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f2866b21f8e04b49bb00257844dedfd3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/09dd5194bebb49cc829b8c48810b0cfb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/10cadf6b35d747a097364e7579cfc504] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=39.3 K 2024-11-14T07:16:15,224 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e81d528c2c4c38987363e6498cc056, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731568571698 2024-11-14T07:16:15,224 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2866b21f8e04b49bb00257844dedfd3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731568571698 2024-11-14T07:16:15,225 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09dd5194bebb49cc829b8c48810b0cfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731568572410 2024-11-14T07:16:15,225 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0281b0a0a3864c5e8cbf7db94f3e6e56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731568572410 2024-11-14T07:16:15,227 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10cadf6b35d747a097364e7579cfc504, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731568573080 2024-11-14T07:16:15,227 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ad209e2d44ab42d4bc3295c1de34764a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731568573080 2024-11-14T07:16:15,247 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:15,248 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/554e9ba2a1c742b8ba99a512a241c314 is 50, key is test_row_0/A:col10/1731568574234/Put/seqid=0 2024-11-14T07:16:15,251 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:15,254 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/c00b0d2989164b8eb444983a16fe2a4d is 50, key is test_row_0/B:col10/1731568574234/Put/seqid=0 2024-11-14T07:16:15,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741921_1097 (size=13255) 2024-11-14T07:16:15,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741922_1098 (size=13255) 2024-11-14T07:16:15,291 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/554e9ba2a1c742b8ba99a512a241c314 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/554e9ba2a1c742b8ba99a512a241c314 2024-11-14T07:16:15,298 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/c00b0d2989164b8eb444983a16fe2a4d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/c00b0d2989164b8eb444983a16fe2a4d 2024-11-14T07:16:15,302 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into 554e9ba2a1c742b8ba99a512a241c314(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:15,302 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:15,302 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=13, startTime=1731568575221; duration=0sec 2024-11-14T07:16:15,302 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:15,302 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:15,302 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:15,309 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:15,309 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:15,309 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,310 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/7e1d3bb6ff014eee86fc7533bb69bad4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/2c4c066a23c5466cb65cf611cf70b553, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/461ad5ef2e5a4fa2baef81841c5b5415] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=36.9 K 2024-11-14T07:16:15,310 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into c00b0d2989164b8eb444983a16fe2a4d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:15,310 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1d3bb6ff014eee86fc7533bb69bad4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731568571698 2024-11-14T07:16:15,310 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:15,310 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=13, startTime=1731568575221; duration=0sec 2024-11-14T07:16:15,311 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:15,311 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c4c066a23c5466cb65cf611cf70b553, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731568572410 2024-11-14T07:16:15,311 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:15,311 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461ad5ef2e5a4fa2baef81841c5b5415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731568573080 2024-11-14T07:16:15,325 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#86 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:15,329 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/6daade5f6a9440d78feecff84c9815a7 is 50, key is test_row_0/C:col10/1731568574234/Put/seqid=0 2024-11-14T07:16:15,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741923_1099 (size=13255) 2024-11-14T07:16:15,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:16:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:15,365 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T07:16:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:15,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/e05a0ec8682f4f4695221663244bdf08 is 50, key is test_row_0/A:col10/1731568574265/Put/seqid=0 2024-11-14T07:16:15,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741924_1100 (size=12301) 2024-11-14T07:16:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:15,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. as already flushing 2024-11-14T07:16:15,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568635447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568635452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:15,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568635562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568635573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:16:15,753 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/6daade5f6a9440d78feecff84c9815a7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/6daade5f6a9440d78feecff84c9815a7 2024-11-14T07:16:15,770 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into 6daade5f6a9440d78feecff84c9815a7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:15,770 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:15,770 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=13, startTime=1731568575222; duration=0sec 2024-11-14T07:16:15,770 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:15,770 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:15,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568635782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:15,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568635782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:15,815 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/e05a0ec8682f4f4695221663244bdf08 2024-11-14T07:16:15,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/ba02f46fc65843d587a8114d00cd19fe is 50, key is test_row_0/B:col10/1731568574265/Put/seqid=0 2024-11-14T07:16:15,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741925_1101 (size=12301) 2024-11-14T07:16:16,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568636099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:16,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568636099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:16,184 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T07:16:16,185 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:16,188 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T07:16:16,188 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:16,188 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T07:16:16,188 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:16,190 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,190 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,190 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,191 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,191 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:16,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:16,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:16,191 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,195 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T07:16:16,195 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:16,195 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,195 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:16,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:16,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/ba02f46fc65843d587a8114d00cd19fe 2024-11-14T07:16:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/38fad058d8fa4951823e9357a376e7e0 is 50, key is test_row_0/C:col10/1731568574265/Put/seqid=0 2024-11-14T07:16:16,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741926_1102 (size=12301) 2024-11-14T07:16:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:16,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43228 deadline: 1731568636612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43186 deadline: 1731568636613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:16,710 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/38fad058d8fa4951823e9357a376e7e0 2024-11-14T07:16:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/e05a0ec8682f4f4695221663244bdf08 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e05a0ec8682f4f4695221663244bdf08 2024-11-14T07:16:16,730 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e05a0ec8682f4f4695221663244bdf08, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T07:16:16,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/ba02f46fc65843d587a8114d00cd19fe as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ba02f46fc65843d587a8114d00cd19fe 2024-11-14T07:16:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:16:16,737 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ba02f46fc65843d587a8114d00cd19fe, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T07:16:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/38fad058d8fa4951823e9357a376e7e0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/38fad058d8fa4951823e9357a376e7e0 2024-11-14T07:16:16,744 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/38fad058d8fa4951823e9357a376e7e0, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T07:16:16,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 349b5cba99ebf7c21762f59b06a8ec03 in 1380ms, sequenceid=574, compaction requested=false 2024-11-14T07:16:16,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:16,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:16,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-14T07:16:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-14T07:16:16,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T07:16:16,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1570 sec 2024-11-14T07:16:16,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.1660 sec 2024-11-14T07:16:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:17,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T07:16:17,634 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T07:16:17,634 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:17,634 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T07:16:17,634 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:17,634 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:17,634 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:17,634 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:17,634 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:17,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:17,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:17,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/264c06b3aa2e4c37a4a324357ea5a615 is 50, key is test_row_0/A:col10/1731568577633/Put/seqid=0 2024-11-14T07:16:17,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741927_1103 (size=12301) 2024-11-14T07:16:18,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/264c06b3aa2e4c37a4a324357ea5a615 2024-11-14T07:16:18,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/6776ca60b67748bcaed18a9322a9841f is 50, key is test_row_0/B:col10/1731568577633/Put/seqid=0 2024-11-14T07:16:18,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741928_1104 (size=12301) 2024-11-14T07:16:18,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/6776ca60b67748bcaed18a9322a9841f 2024-11-14T07:16:18,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f8e2657be3d9486e9ae3caff7d7915c1 is 50, key is test_row_0/C:col10/1731568577633/Put/seqid=0 2024-11-14T07:16:18,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741929_1105 (size=12301) 2024-11-14T07:16:18,496 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:16:18,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:18,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f8e2657be3d9486e9ae3caff7d7915c1 2024-11-14T07:16:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/264c06b3aa2e4c37a4a324357ea5a615 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/264c06b3aa2e4c37a4a324357ea5a615 2024-11-14T07:16:18,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/264c06b3aa2e4c37a4a324357ea5a615, entries=150, sequenceid=591, filesize=12.0 K 2024-11-14T07:16:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/6776ca60b67748bcaed18a9322a9841f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6776ca60b67748bcaed18a9322a9841f 2024-11-14T07:16:18,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6776ca60b67748bcaed18a9322a9841f, entries=150, sequenceid=591, filesize=12.0 K 2024-11-14T07:16:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/f8e2657be3d9486e9ae3caff7d7915c1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f8e2657be3d9486e9ae3caff7d7915c1 2024-11-14T07:16:18,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f8e2657be3d9486e9ae3caff7d7915c1, entries=150, sequenceid=591, filesize=12.0 K 2024-11-14T07:16:18,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 349b5cba99ebf7c21762f59b06a8ec03 in 1279ms, sequenceid=591, compaction requested=true 2024-11-14T07:16:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:18,914 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 349b5cba99ebf7c21762f59b06a8ec03:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:16:18,914 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:18,915 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:18,915 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:18,915 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/A is initiating minor compaction (all files) 2024-11-14T07:16:18,915 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/B is initiating minor compaction (all files) 2024-11-14T07:16:18,915 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/B in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:18,915 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/A in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:18,915 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/554e9ba2a1c742b8ba99a512a241c314, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e05a0ec8682f4f4695221663244bdf08, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/264c06b3aa2e4c37a4a324357ea5a615] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=37.0 K 2024-11-14T07:16:18,915 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/c00b0d2989164b8eb444983a16fe2a4d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ba02f46fc65843d587a8114d00cd19fe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6776ca60b67748bcaed18a9322a9841f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=37.0 K 2024-11-14T07:16:18,916 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 554e9ba2a1c742b8ba99a512a241c314, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731568573080 2024-11-14T07:16:18,916 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c00b0d2989164b8eb444983a16fe2a4d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731568573080 2024-11-14T07:16:18,916 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting e05a0ec8682f4f4695221663244bdf08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731568574265 2024-11-14T07:16:18,916 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ba02f46fc65843d587a8114d00cd19fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731568574265 2024-11-14T07:16:18,917 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264c06b3aa2e4c37a4a324357ea5a615, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731568575445 2024-11-14T07:16:18,917 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6776ca60b67748bcaed18a9322a9841f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731568575445 2024-11-14T07:16:18,929 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:18,930 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/679b679687874863947d0346bcf15021 is 50, key is test_row_0/A:col10/1731568577633/Put/seqid=0 2024-11-14T07:16:18,934 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#B#compaction#94 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:18,934 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/59a35681b869445fb9ac29cd6e8eb464 is 50, key is test_row_0/B:col10/1731568577633/Put/seqid=0 2024-11-14T07:16:18,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741930_1106 (size=13357) 2024-11-14T07:16:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741931_1107 (size=13357) 2024-11-14T07:16:19,343 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/679b679687874863947d0346bcf15021 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/679b679687874863947d0346bcf15021 2024-11-14T07:16:19,346 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/59a35681b869445fb9ac29cd6e8eb464 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/59a35681b869445fb9ac29cd6e8eb464 2024-11-14T07:16:19,349 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/A of 349b5cba99ebf7c21762f59b06a8ec03 into 679b679687874863947d0346bcf15021(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:19,349 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:19,349 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/A, priority=13, startTime=1731568578914; duration=0sec 2024-11-14T07:16:19,349 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:19,349 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:A 2024-11-14T07:16:19,349 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:19,350 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:19,350 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 349b5cba99ebf7c21762f59b06a8ec03/C is initiating minor compaction (all files) 2024-11-14T07:16:19,350 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 349b5cba99ebf7c21762f59b06a8ec03/C in TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:19,351 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/6daade5f6a9440d78feecff84c9815a7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/38fad058d8fa4951823e9357a376e7e0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f8e2657be3d9486e9ae3caff7d7915c1] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp, totalSize=37.0 K 2024-11-14T07:16:19,351 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6daade5f6a9440d78feecff84c9815a7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731568573080 2024-11-14T07:16:19,351 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38fad058d8fa4951823e9357a376e7e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731568574265 2024-11-14T07:16:19,352 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8e2657be3d9486e9ae3caff7d7915c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731568575445 2024-11-14T07:16:19,352 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/B of 349b5cba99ebf7c21762f59b06a8ec03 into 59a35681b869445fb9ac29cd6e8eb464(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:19,352 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:19,352 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/B, priority=13, startTime=1731568578914; duration=0sec 2024-11-14T07:16:19,352 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:19,352 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:B 2024-11-14T07:16:19,361 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 349b5cba99ebf7c21762f59b06a8ec03#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:19,362 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/3d0380b199c9434da3506d2fe8d1cf69 is 50, key is test_row_0/C:col10/1731568577633/Put/seqid=0 2024-11-14T07:16:19,363 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T07:16:19,363 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:19,363 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,363 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:19,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741932_1108 (size=13357) 2024-11-14T07:16:19,403 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T07:16:19,403 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:19,403 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,404 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,404 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:19,414 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T07:16:19,414 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:19,414 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,414 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,414 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:19,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:16:19,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-14T07:16:19,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-14T07:16:19,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 205 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 178 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2518 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2433 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1103 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3307 rows 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1114 2024-11-14T07:16:19,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3338 rows 2024-11-14T07:16:19,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:16:19,415 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:16:19,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:19,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:19,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:16:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:16:19,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568579429"}]},"ts":"1731568579429"} 2024-11-14T07:16:19,431 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:16:19,431 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:16:19,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:16:19,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, UNASSIGN}] 2024-11-14T07:16:19,437 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, UNASSIGN 2024-11-14T07:16:19,439 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=349b5cba99ebf7c21762f59b06a8ec03, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:19,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, UNASSIGN because future has completed 2024-11-14T07:16:19,442 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:16:19,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:16:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:16:19,600 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:19,600 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:16:19,601 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 349b5cba99ebf7c21762f59b06a8ec03, disabling compactions & flushes 2024-11-14T07:16:19,602 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:16:19,772 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/3d0380b199c9434da3506d2fe8d1cf69 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3d0380b199c9434da3506d2fe8d1cf69 2024-11-14T07:16:19,777 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 349b5cba99ebf7c21762f59b06a8ec03/C of 349b5cba99ebf7c21762f59b06a8ec03 into 3d0380b199c9434da3506d2fe8d1cf69(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:19,778 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 349b5cba99ebf7c21762f59b06a8ec03: 2024-11-14T07:16:19,778 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:19,778 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., storeName=349b5cba99ebf7c21762f59b06a8ec03/C, priority=13, startTime=1731568578914; duration=0sec 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:19,778 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. after waiting 0 ms 2024-11-14T07:16:19,778 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 349b5cba99ebf7c21762f59b06a8ec03:C 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:19,778 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 349b5cba99ebf7c21762f59b06a8ec03 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=A 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=B 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 349b5cba99ebf7c21762f59b06a8ec03, store=C 2024-11-14T07:16:19,778 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:19,782 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/21c62f5884624339bbf94c2cbd0d5ac0 is 50, key is test_row_0/A:col10/1731568579412/Put/seqid=0 2024-11-14T07:16:19,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741933_1109 (size=9857) 2024-11-14T07:16:19,790 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/21c62f5884624339bbf94c2cbd0d5ac0 2024-11-14T07:16:19,798 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/1341ebc5eae5417aa375f903f88d1c8c is 50, key is test_row_0/B:col10/1731568579412/Put/seqid=0 2024-11-14T07:16:19,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741934_1110 (size=9857) 2024-11-14T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:16:20,203 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/1341ebc5eae5417aa375f903f88d1c8c 2024-11-14T07:16:20,214 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/9fce748eb0d24080a24ce8d53c6e0d19 is 50, key is test_row_0/C:col10/1731568579412/Put/seqid=0 2024-11-14T07:16:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741935_1111 (size=9857) 2024-11-14T07:16:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:16:20,618 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/9fce748eb0d24080a24ce8d53c6e0d19 2024-11-14T07:16:20,623 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/A/21c62f5884624339bbf94c2cbd0d5ac0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/21c62f5884624339bbf94c2cbd0d5ac0 2024-11-14T07:16:20,628 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/21c62f5884624339bbf94c2cbd0d5ac0, entries=100, sequenceid=600, filesize=9.6 K 2024-11-14T07:16:20,629 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/B/1341ebc5eae5417aa375f903f88d1c8c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/1341ebc5eae5417aa375f903f88d1c8c 2024-11-14T07:16:20,634 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/1341ebc5eae5417aa375f903f88d1c8c, entries=100, sequenceid=600, filesize=9.6 K 2024-11-14T07:16:20,636 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/.tmp/C/9fce748eb0d24080a24ce8d53c6e0d19 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/9fce748eb0d24080a24ce8d53c6e0d19 2024-11-14T07:16:20,640 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/9fce748eb0d24080a24ce8d53c6e0d19, entries=100, sequenceid=600, filesize=9.6 K 2024-11-14T07:16:20,641 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 349b5cba99ebf7c21762f59b06a8ec03 in 863ms, sequenceid=600, compaction requested=false 2024-11-14T07:16:20,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ce533cced4df44ee877b9d66b2e0f90b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2aa64732fef54dcb834dc1934909bb83, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5c4b7b3aa67b476fb7329b3521e52c88, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/c9c1f9fb81234c10aa9fce0fd899a008, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5503aecb8ba34fe7af5ef96c2f110644, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b9d2fe3fa4aa4a09a0936950a46544bc, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/305b80bd54f64de4b53835f1dda980f9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/d486d2d9d59f4b41937cb9c4caef4c14, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6d775e00fb9541a1b7d86ca7a98955ea, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ebab44eeee5c4637ba17715af1c99c66, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/a2a42ccb823e4d4ca463f8d0c8781df0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/1eb5933fd17a4e61b312e5f1bfa272e7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/dc4721307f4c47d8b64456829a60bb6a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2ff42247a96946ab87ff8fa08f5d8cf9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/13b6bd00eb1e4032a39975ccf6af18d4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ee56767e5de1403492ad3892fce06e9e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ed11af2bad9f4cd6a95e3401f7119908, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/eebc316b35ef4a618b599f462a24e806, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e6b9340d2c634c2389415cd352840463, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6cebf50fa5184153b135ff3833c2f5fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/40489e4da6814941bbb6ca8b7fd4e258, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f6a59db667ea449298de999d4faacca2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b358825caf2e40d097f611cf1bdfc4eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ac2c818ce0bf49e18b043e3cf8baaade, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/343b728df1664be59401d158455ca3ca, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f2866b21f8e04b49bb00257844dedfd3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/09dd5194bebb49cc829b8c48810b0cfb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/10cadf6b35d747a097364e7579cfc504, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/554e9ba2a1c742b8ba99a512a241c314, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e05a0ec8682f4f4695221663244bdf08, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/264c06b3aa2e4c37a4a324357ea5a615] to archive 2024-11-14T07:16:20,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:16:20,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ce533cced4df44ee877b9d66b2e0f90b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ce533cced4df44ee877b9d66b2e0f90b 2024-11-14T07:16:20,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2aa64732fef54dcb834dc1934909bb83 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2aa64732fef54dcb834dc1934909bb83 2024-11-14T07:16:20,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5c4b7b3aa67b476fb7329b3521e52c88 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5c4b7b3aa67b476fb7329b3521e52c88 2024-11-14T07:16:20,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/c9c1f9fb81234c10aa9fce0fd899a008 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/c9c1f9fb81234c10aa9fce0fd899a008 2024-11-14T07:16:20,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5503aecb8ba34fe7af5ef96c2f110644 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/5503aecb8ba34fe7af5ef96c2f110644 2024-11-14T07:16:20,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b9d2fe3fa4aa4a09a0936950a46544bc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b9d2fe3fa4aa4a09a0936950a46544bc 2024-11-14T07:16:20,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/305b80bd54f64de4b53835f1dda980f9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/305b80bd54f64de4b53835f1dda980f9 2024-11-14T07:16:20,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/d486d2d9d59f4b41937cb9c4caef4c14 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/d486d2d9d59f4b41937cb9c4caef4c14 2024-11-14T07:16:20,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6d775e00fb9541a1b7d86ca7a98955ea to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6d775e00fb9541a1b7d86ca7a98955ea 2024-11-14T07:16:20,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ebab44eeee5c4637ba17715af1c99c66 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ebab44eeee5c4637ba17715af1c99c66 2024-11-14T07:16:20,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/a2a42ccb823e4d4ca463f8d0c8781df0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/a2a42ccb823e4d4ca463f8d0c8781df0 2024-11-14T07:16:20,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/1eb5933fd17a4e61b312e5f1bfa272e7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/1eb5933fd17a4e61b312e5f1bfa272e7 2024-11-14T07:16:20,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/dc4721307f4c47d8b64456829a60bb6a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/dc4721307f4c47d8b64456829a60bb6a 2024-11-14T07:16:20,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2ff42247a96946ab87ff8fa08f5d8cf9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/2ff42247a96946ab87ff8fa08f5d8cf9 2024-11-14T07:16:20,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/13b6bd00eb1e4032a39975ccf6af18d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/13b6bd00eb1e4032a39975ccf6af18d4 2024-11-14T07:16:20,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ee56767e5de1403492ad3892fce06e9e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ee56767e5de1403492ad3892fce06e9e 2024-11-14T07:16:20,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ed11af2bad9f4cd6a95e3401f7119908 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ed11af2bad9f4cd6a95e3401f7119908 2024-11-14T07:16:20,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/eebc316b35ef4a618b599f462a24e806 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/eebc316b35ef4a618b599f462a24e806 2024-11-14T07:16:20,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e6b9340d2c634c2389415cd352840463 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e6b9340d2c634c2389415cd352840463 2024-11-14T07:16:20,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6cebf50fa5184153b135ff3833c2f5fb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/6cebf50fa5184153b135ff3833c2f5fb 2024-11-14T07:16:20,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/40489e4da6814941bbb6ca8b7fd4e258 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/40489e4da6814941bbb6ca8b7fd4e258 2024-11-14T07:16:20,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f6a59db667ea449298de999d4faacca2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f6a59db667ea449298de999d4faacca2 2024-11-14T07:16:20,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b358825caf2e40d097f611cf1bdfc4eb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/b358825caf2e40d097f611cf1bdfc4eb 2024-11-14T07:16:20,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ac2c818ce0bf49e18b043e3cf8baaade to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/ac2c818ce0bf49e18b043e3cf8baaade 2024-11-14T07:16:20,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/343b728df1664be59401d158455ca3ca to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/343b728df1664be59401d158455ca3ca 2024-11-14T07:16:20,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f2866b21f8e04b49bb00257844dedfd3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/f2866b21f8e04b49bb00257844dedfd3 2024-11-14T07:16:20,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/09dd5194bebb49cc829b8c48810b0cfb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/09dd5194bebb49cc829b8c48810b0cfb 2024-11-14T07:16:20,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/10cadf6b35d747a097364e7579cfc504 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/10cadf6b35d747a097364e7579cfc504 2024-11-14T07:16:20,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/554e9ba2a1c742b8ba99a512a241c314 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/554e9ba2a1c742b8ba99a512a241c314 2024-11-14T07:16:20,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e05a0ec8682f4f4695221663244bdf08 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/e05a0ec8682f4f4695221663244bdf08 2024-11-14T07:16:20,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/264c06b3aa2e4c37a4a324357ea5a615 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/264c06b3aa2e4c37a4a324357ea5a615 2024-11-14T07:16:20,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a902e75d68be43aabe8dfb52527149db, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63776d3e48354ecd9d5f5f4df3d8a5d2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/bb42e219d6d349ca8934a70d2b072fbb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a09c13de5d3c4a0fa528702e1e5e47c5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5d86c2d8ebb940248af19bb717e28c03, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/4e3a6781c2084374b947459cb7cb70f6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/40de5dff305b4cba92d1c8052a39e024, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/66f30df076844addadd6ca55680c50c8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/757de47b84d44bf692b9238cd4fce127, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6253a463c3bb44cdb613fb92ecc165aa, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9af6071cfe0e4a469cfe579d317b6b32, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/2244f704dfef49e18474f580d5367215, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/fe60ab50e39449cd9827237b5743ebef, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5c16a914767a42ca979f55a300b13360, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/09dd8d947cb245a3b6fc6e1559ac5d24, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/8956ee23fe6a4e198d5449a27a2d29b6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/abaaae37931646b4bd8acda37bd9b314, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/f7a62e2957e84b438cb4e7fe05f747fa, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63e5438817da4fcd82d0d9e4400109f6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/77838e47f38449918fd52d9c12273000, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/27ee877c976245d18a71f3d040c9a209, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/847ded74d6b54a3cad0a27b85edfb67f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/db17bbcdd7ef469681ef424d2c77881c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9fae37defb9e488cbc796862f070add8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/10e81d528c2c4c38987363e6498cc056, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/d6612505267140ff81c0c0d3ca5942a5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/0281b0a0a3864c5e8cbf7db94f3e6e56, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/c00b0d2989164b8eb444983a16fe2a4d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ad209e2d44ab42d4bc3295c1de34764a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ba02f46fc65843d587a8114d00cd19fe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6776ca60b67748bcaed18a9322a9841f] to archive 2024-11-14T07:16:20,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:16:20,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a902e75d68be43aabe8dfb52527149db to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a902e75d68be43aabe8dfb52527149db 2024-11-14T07:16:20,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63776d3e48354ecd9d5f5f4df3d8a5d2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63776d3e48354ecd9d5f5f4df3d8a5d2 2024-11-14T07:16:20,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/bb42e219d6d349ca8934a70d2b072fbb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/bb42e219d6d349ca8934a70d2b072fbb 2024-11-14T07:16:20,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a09c13de5d3c4a0fa528702e1e5e47c5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/a09c13de5d3c4a0fa528702e1e5e47c5 2024-11-14T07:16:20,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5d86c2d8ebb940248af19bb717e28c03 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5d86c2d8ebb940248af19bb717e28c03 2024-11-14T07:16:20,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/4e3a6781c2084374b947459cb7cb70f6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/4e3a6781c2084374b947459cb7cb70f6 2024-11-14T07:16:20,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/40de5dff305b4cba92d1c8052a39e024 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/40de5dff305b4cba92d1c8052a39e024 2024-11-14T07:16:20,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/66f30df076844addadd6ca55680c50c8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/66f30df076844addadd6ca55680c50c8 2024-11-14T07:16:20,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/757de47b84d44bf692b9238cd4fce127 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/757de47b84d44bf692b9238cd4fce127 2024-11-14T07:16:20,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6253a463c3bb44cdb613fb92ecc165aa to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6253a463c3bb44cdb613fb92ecc165aa 2024-11-14T07:16:20,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9af6071cfe0e4a469cfe579d317b6b32 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9af6071cfe0e4a469cfe579d317b6b32 2024-11-14T07:16:20,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/2244f704dfef49e18474f580d5367215 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/2244f704dfef49e18474f580d5367215 2024-11-14T07:16:20,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/fe60ab50e39449cd9827237b5743ebef to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/fe60ab50e39449cd9827237b5743ebef 2024-11-14T07:16:20,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5c16a914767a42ca979f55a300b13360 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/5c16a914767a42ca979f55a300b13360 2024-11-14T07:16:20,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/09dd8d947cb245a3b6fc6e1559ac5d24 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/09dd8d947cb245a3b6fc6e1559ac5d24 2024-11-14T07:16:20,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/8956ee23fe6a4e198d5449a27a2d29b6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/8956ee23fe6a4e198d5449a27a2d29b6 2024-11-14T07:16:20,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/abaaae37931646b4bd8acda37bd9b314 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/abaaae37931646b4bd8acda37bd9b314 2024-11-14T07:16:20,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/f7a62e2957e84b438cb4e7fe05f747fa to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/f7a62e2957e84b438cb4e7fe05f747fa 2024-11-14T07:16:20,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63e5438817da4fcd82d0d9e4400109f6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/63e5438817da4fcd82d0d9e4400109f6 2024-11-14T07:16:20,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/77838e47f38449918fd52d9c12273000 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/77838e47f38449918fd52d9c12273000 2024-11-14T07:16:20,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/27ee877c976245d18a71f3d040c9a209 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/27ee877c976245d18a71f3d040c9a209 2024-11-14T07:16:20,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/847ded74d6b54a3cad0a27b85edfb67f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/847ded74d6b54a3cad0a27b85edfb67f 2024-11-14T07:16:20,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/db17bbcdd7ef469681ef424d2c77881c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/db17bbcdd7ef469681ef424d2c77881c 2024-11-14T07:16:20,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9fae37defb9e488cbc796862f070add8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/9fae37defb9e488cbc796862f070add8 2024-11-14T07:16:20,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/10e81d528c2c4c38987363e6498cc056 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/10e81d528c2c4c38987363e6498cc056 2024-11-14T07:16:20,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/d6612505267140ff81c0c0d3ca5942a5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/d6612505267140ff81c0c0d3ca5942a5 2024-11-14T07:16:20,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/0281b0a0a3864c5e8cbf7db94f3e6e56 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/0281b0a0a3864c5e8cbf7db94f3e6e56 2024-11-14T07:16:20,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/c00b0d2989164b8eb444983a16fe2a4d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/c00b0d2989164b8eb444983a16fe2a4d 2024-11-14T07:16:20,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ad209e2d44ab42d4bc3295c1de34764a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ad209e2d44ab42d4bc3295c1de34764a 2024-11-14T07:16:20,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ba02f46fc65843d587a8114d00cd19fe to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/ba02f46fc65843d587a8114d00cd19fe 2024-11-14T07:16:20,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6776ca60b67748bcaed18a9322a9841f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/6776ca60b67748bcaed18a9322a9841f 2024-11-14T07:16:20,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cf2efe853cfe41a5b7ec2a335b27ac43, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f329b8302ba3493c9716cbd7824a52d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/048873d7f55047808f819454832fa7ce, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f53b757a2c85411789c82a5472a5e8f7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d360389e38cb4942a20cc3428f0d4761, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/10f2e80afc1c41b0920d0b59a293f182, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/99b9f2b02a264a2aaca3cf6ee4c33bb5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/0a98be88cda94e988071b761f36a2ae4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d270aa32447b4c2e823d4f257a3e89a5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a3424c78206549a0a288dbd3d9ba94a5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/1b5cbcb9118940ea82ec1e3e3f27c31b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/24552f08d61c478ba1f9150719cb773c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/456fcbf813b84dd3988cabe957510623, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3be77a434e6e4437ae659e89d611681a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/928e88d04f58453d9d5cf245ff3e0ccb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fa28ecf915b448dfb8df8bf7fad184df, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fcc093bb9791494d900e907e8697c364, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/de8a281b9316466db83d20c4922a599f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8a05ea4aa6b40d3b91eeff20126b730, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a96b31b4498b4a2d8a1bc29fd42469d9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8b46ce0921e4748b616f945f893091a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/17a70d21ba08467e93469f507e1dda7b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/92e4c716eba14d1c8a3a6d35c6ba5e1b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cca0e12a3d254b62b26804d465f8cfd6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/7e1d3bb6ff014eee86fc7533bb69bad4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a84bfb73040e410683eb4cf92b231bbb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/2c4c066a23c5466cb65cf611cf70b553, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/6daade5f6a9440d78feecff84c9815a7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/461ad5ef2e5a4fa2baef81841c5b5415, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/38fad058d8fa4951823e9357a376e7e0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f8e2657be3d9486e9ae3caff7d7915c1] to archive 2024-11-14T07:16:20,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:16:20,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cf2efe853cfe41a5b7ec2a335b27ac43 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cf2efe853cfe41a5b7ec2a335b27ac43 2024-11-14T07:16:20,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f329b8302ba3493c9716cbd7824a52d8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f329b8302ba3493c9716cbd7824a52d8 2024-11-14T07:16:20,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/048873d7f55047808f819454832fa7ce to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/048873d7f55047808f819454832fa7ce 2024-11-14T07:16:20,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f53b757a2c85411789c82a5472a5e8f7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f53b757a2c85411789c82a5472a5e8f7 2024-11-14T07:16:20,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d360389e38cb4942a20cc3428f0d4761 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d360389e38cb4942a20cc3428f0d4761 2024-11-14T07:16:20,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/10f2e80afc1c41b0920d0b59a293f182 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/10f2e80afc1c41b0920d0b59a293f182 2024-11-14T07:16:20,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/99b9f2b02a264a2aaca3cf6ee4c33bb5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/99b9f2b02a264a2aaca3cf6ee4c33bb5 2024-11-14T07:16:20,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/0a98be88cda94e988071b761f36a2ae4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/0a98be88cda94e988071b761f36a2ae4 2024-11-14T07:16:20,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d270aa32447b4c2e823d4f257a3e89a5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d270aa32447b4c2e823d4f257a3e89a5 2024-11-14T07:16:20,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a3424c78206549a0a288dbd3d9ba94a5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a3424c78206549a0a288dbd3d9ba94a5 2024-11-14T07:16:20,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/1b5cbcb9118940ea82ec1e3e3f27c31b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/1b5cbcb9118940ea82ec1e3e3f27c31b 2024-11-14T07:16:20,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/24552f08d61c478ba1f9150719cb773c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/24552f08d61c478ba1f9150719cb773c 2024-11-14T07:16:20,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/456fcbf813b84dd3988cabe957510623 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/456fcbf813b84dd3988cabe957510623 2024-11-14T07:16:20,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3be77a434e6e4437ae659e89d611681a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3be77a434e6e4437ae659e89d611681a 2024-11-14T07:16:20,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/928e88d04f58453d9d5cf245ff3e0ccb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/928e88d04f58453d9d5cf245ff3e0ccb 2024-11-14T07:16:20,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fa28ecf915b448dfb8df8bf7fad184df to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fa28ecf915b448dfb8df8bf7fad184df 2024-11-14T07:16:20,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fcc093bb9791494d900e907e8697c364 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/fcc093bb9791494d900e907e8697c364 2024-11-14T07:16:20,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/de8a281b9316466db83d20c4922a599f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/de8a281b9316466db83d20c4922a599f 2024-11-14T07:16:20,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8a05ea4aa6b40d3b91eeff20126b730 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8a05ea4aa6b40d3b91eeff20126b730 2024-11-14T07:16:20,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a96b31b4498b4a2d8a1bc29fd42469d9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a96b31b4498b4a2d8a1bc29fd42469d9 2024-11-14T07:16:20,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8b46ce0921e4748b616f945f893091a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/d8b46ce0921e4748b616f945f893091a 2024-11-14T07:16:20,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/17a70d21ba08467e93469f507e1dda7b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/17a70d21ba08467e93469f507e1dda7b 2024-11-14T07:16:20,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/92e4c716eba14d1c8a3a6d35c6ba5e1b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/92e4c716eba14d1c8a3a6d35c6ba5e1b 2024-11-14T07:16:20,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cca0e12a3d254b62b26804d465f8cfd6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/cca0e12a3d254b62b26804d465f8cfd6 2024-11-14T07:16:20,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/7e1d3bb6ff014eee86fc7533bb69bad4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/7e1d3bb6ff014eee86fc7533bb69bad4 2024-11-14T07:16:20,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a84bfb73040e410683eb4cf92b231bbb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/a84bfb73040e410683eb4cf92b231bbb 2024-11-14T07:16:20,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/2c4c066a23c5466cb65cf611cf70b553 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/2c4c066a23c5466cb65cf611cf70b553 2024-11-14T07:16:20,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/6daade5f6a9440d78feecff84c9815a7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/6daade5f6a9440d78feecff84c9815a7 2024-11-14T07:16:20,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/461ad5ef2e5a4fa2baef81841c5b5415 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/461ad5ef2e5a4fa2baef81841c5b5415 2024-11-14T07:16:20,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/38fad058d8fa4951823e9357a376e7e0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/38fad058d8fa4951823e9357a376e7e0 2024-11-14T07:16:20,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f8e2657be3d9486e9ae3caff7d7915c1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/f8e2657be3d9486e9ae3caff7d7915c1 2024-11-14T07:16:20,773 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/recovered.edits/603.seqid, newMaxSeqId=603, maxSeqId=1 2024-11-14T07:16:20,773 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. 2024-11-14T07:16:20,773 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 349b5cba99ebf7c21762f59b06a8ec03: Waiting for close lock at 1731568579601Running coprocessor pre-close hooks at 1731568579601Disabling compacts and flushes for region at 1731568579601Disabling writes for close at 1731568579778 (+177 ms)Obtaining lock to block concurrent updates at 1731568579778Preparing flush snapshotting stores in 349b5cba99ebf7c21762f59b06a8ec03 at 1731568579778Finished memstore snapshotting TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731568579778Flushing stores of TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03. at 1731568579779 (+1 ms)Flushing 349b5cba99ebf7c21762f59b06a8ec03/A: creating writer at 1731568579779Flushing 349b5cba99ebf7c21762f59b06a8ec03/A: appending metadata at 1731568579782 (+3 ms)Flushing 349b5cba99ebf7c21762f59b06a8ec03/A: closing flushed file at 1731568579782Flushing 349b5cba99ebf7c21762f59b06a8ec03/B: creating writer at 1731568579794 (+12 ms)Flushing 349b5cba99ebf7c21762f59b06a8ec03/B: appending metadata at 1731568579797 (+3 ms)Flushing 349b5cba99ebf7c21762f59b06a8ec03/B: closing flushed file at 1731568579797Flushing 349b5cba99ebf7c21762f59b06a8ec03/C: creating writer at 1731568580210 (+413 ms)Flushing 349b5cba99ebf7c21762f59b06a8ec03/C: appending metadata at 1731568580213 (+3 ms)Flushing 349b5cba99ebf7c21762f59b06a8ec03/C: closing flushed file at 1731568580213Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e710a13: reopening flushed file at 1731568580622 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a665223: reopening flushed file at 1731568580628 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e853ba0: reopening flushed file at 1731568580635 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 349b5cba99ebf7c21762f59b06a8ec03 in 863ms, sequenceid=600, compaction requested=false at 1731568580641 (+6 ms)Writing region close event to WAL at 1731568580770 (+129 ms)Running coprocessor post-close hooks at 1731568580773 (+3 ms)Closed at 1731568580773 2024-11-14T07:16:20,776 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:20,776 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=349b5cba99ebf7c21762f59b06a8ec03, regionState=CLOSED 2024-11-14T07:16:20,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:16:20,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T07:16:20,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 349b5cba99ebf7c21762f59b06a8ec03, server=c7aa6e2790b3,33845,1731568550163 in 1.3360 sec 2024-11-14T07:16:20,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-14T07:16:20,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=349b5cba99ebf7c21762f59b06a8ec03, UNASSIGN in 1.3460 sec 2024-11-14T07:16:20,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T07:16:20,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3530 sec 2024-11-14T07:16:20,788 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568580788"}]},"ts":"1731568580788"} 2024-11-14T07:16:20,790 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:16:20,790 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:16:20,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3680 sec 2024-11-14T07:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:16:21,582 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:21,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:21,590 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-14T07:16:21,592 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:21,597 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:21,603 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/recovered.edits] 2024-11-14T07:16:21,608 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/21c62f5884624339bbf94c2cbd0d5ac0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/21c62f5884624339bbf94c2cbd0d5ac0 2024-11-14T07:16:21,610 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/679b679687874863947d0346bcf15021 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/A/679b679687874863947d0346bcf15021 2024-11-14T07:16:21,613 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/1341ebc5eae5417aa375f903f88d1c8c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/1341ebc5eae5417aa375f903f88d1c8c 2024-11-14T07:16:21,614 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/59a35681b869445fb9ac29cd6e8eb464 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/B/59a35681b869445fb9ac29cd6e8eb464 2024-11-14T07:16:21,617 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3d0380b199c9434da3506d2fe8d1cf69 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/3d0380b199c9434da3506d2fe8d1cf69 2024-11-14T07:16:21,618 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/9fce748eb0d24080a24ce8d53c6e0d19 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/C/9fce748eb0d24080a24ce8d53c6e0d19 2024-11-14T07:16:21,621 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/recovered.edits/603.seqid to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03/recovered.edits/603.seqid 2024-11-14T07:16:21,622 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/349b5cba99ebf7c21762f59b06a8ec03 2024-11-14T07:16:21,622 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:16:21,627 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:21,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T07:16:21,636 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:16:21,665 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:16:21,667 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:21,667 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:16:21,667 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731568581667"}]},"ts":"9223372036854775807"} 2024-11-14T07:16:21,671 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:16:21,671 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 349b5cba99ebf7c21762f59b06a8ec03, NAME => 'TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:16:21,671 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:16:21,672 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731568581671"}]},"ts":"9223372036854775807"} 2024-11-14T07:16:21,675 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:16:21,676 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:21,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-14T07:16:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-14T07:16:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:16:21,703 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:21,712 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1006931501_22 at /127.0.0.1:48634 [Waiting for operation #175] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c7aa6e2790b3:33845-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-406272114_22 at /127.0.0.1:33898 [Waiting for operation #227] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-406272114_22 at /127.0.0.1:48522 [Waiting for operation #203] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1006931501_22 at /127.0.0.1:45038 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 336) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5865 (was 7158) 2024-11-14T07:16:21,721 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=5862 2024-11-14T07:16:21,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:16:21,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:16:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:21,726 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:16:21,726 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:21,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-14T07:16:21,728 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:16:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T07:16:21,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741936_1112 (size=1061) 2024-11-14T07:16:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T07:16:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T07:16:22,137 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:16:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741937_1113 (size=53) 2024-11-14T07:16:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T07:16:22,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:16:22,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 26fca7a5cd511ec9993090c4fc5431d4, disabling compactions & flushes 2024-11-14T07:16:22,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. after waiting 0 ms 2024-11-14T07:16:22,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 26fca7a5cd511ec9993090c4fc5431d4: Waiting for close lock at 1731568582544Disabling compacts and flushes for region at 1731568582544Disabling writes for close at 1731568582544Writing region close event to WAL at 1731568582544Closed at 1731568582544 2024-11-14T07:16:22,546 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:16:22,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731568582546"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731568582546"}]},"ts":"1731568582546"} 2024-11-14T07:16:22,548 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:16:22,549 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:16:22,550 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568582549"}]},"ts":"1731568582549"} 2024-11-14T07:16:22,551 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:16:22,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, ASSIGN}] 2024-11-14T07:16:22,553 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, ASSIGN 2024-11-14T07:16:22,554 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=false 2024-11-14T07:16:22,705 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:22,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, ASSIGN because future has completed 2024-11-14T07:16:22,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:16:22,865 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,865 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:16:22,865 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,865 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:16:22,866 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,866 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,868 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,870 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:22,871 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fca7a5cd511ec9993090c4fc5431d4 columnFamilyName A 2024-11-14T07:16:22,872 DEBUG [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T07:16:22,873 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(327): Store=26fca7a5cd511ec9993090c4fc5431d4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:22,873 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,875 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:22,876 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fca7a5cd511ec9993090c4fc5431d4 columnFamilyName B 2024-11-14T07:16:22,876 DEBUG [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:22,877 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(327): Store=26fca7a5cd511ec9993090c4fc5431d4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:22,877 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,879 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:22,879 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fca7a5cd511ec9993090c4fc5431d4 columnFamilyName C 2024-11-14T07:16:22,879 DEBUG [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:22,879 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(327): Store=26fca7a5cd511ec9993090c4fc5431d4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:22,879 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,880 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,880 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,881 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,882 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,882 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,882 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:16:22,883 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,885 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:16:22,885 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 26fca7a5cd511ec9993090c4fc5431d4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63191050, jitterRate=-0.05837997794151306}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:16:22,885 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:22,886 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 26fca7a5cd511ec9993090c4fc5431d4: Running coprocessor pre-open hook at 1731568582866Writing region info on filesystem at 1731568582866Initializing all the Stores at 1731568582867 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568582867Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568582867Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568582867Cleaning up temporary data from old regions at 1731568582882 (+15 ms)Running coprocessor post-open hooks at 1731568582885 (+3 ms)Region opened successfully at 1731568582886 (+1 ms) 2024-11-14T07:16:22,887 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., pid=34, masterSystemTime=1731568582860 2024-11-14T07:16:22,888 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,888 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:22,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:22,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:16:22,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-14T07:16:22,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 in 184 msec 2024-11-14T07:16:22,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-14T07:16:22,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, ASSIGN in 343 msec 2024-11-14T07:16:22,897 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:16:22,897 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568582897"}]},"ts":"1731568582897"} 2024-11-14T07:16:22,899 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:16:22,900 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:16:22,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-14T07:16:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T07:16:23,882 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:23,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:23,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:23,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:23,887 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4caf3862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:23,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2832b027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:23,893 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:23,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:23,899 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:16:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,37073,1731568549456 2024-11-14T07:16:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48726c18 2024-11-14T07:16:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:16:23,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:16:23,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:16:23,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:16:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:23,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741938_1114 (size=1097) 2024-11-14T07:16:24,327 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T07:16:24,327 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T07:16:24,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:16:24,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, REOPEN/MOVE}] 2024-11-14T07:16:24,343 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, REOPEN/MOVE 2024-11-14T07:16:24,344 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:24,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, REOPEN/MOVE because future has completed 2024-11-14T07:16:24,346 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:16:24,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:16:24,500 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,500 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:16:24,500 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 26fca7a5cd511ec9993090c4fc5431d4, disabling compactions & flushes 2024-11-14T07:16:24,500 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,500 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,500 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. after waiting 0 ms 2024-11-14T07:16:24,500 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,503 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T07:16:24,504 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,504 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 26fca7a5cd511ec9993090c4fc5431d4: Waiting for close lock at 1731568584500Running coprocessor pre-close hooks at 1731568584500Disabling compacts and flushes for region at 1731568584500Disabling writes for close at 1731568584500Writing region close event to WAL at 1731568584500Running coprocessor post-close hooks at 1731568584504 (+4 ms)Closed at 1731568584504 2024-11-14T07:16:24,504 WARN [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 26fca7a5cd511ec9993090c4fc5431d4 to self. 2024-11-14T07:16:24,506 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,507 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=CLOSED 2024-11-14T07:16:24,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:16:24,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-14T07:16:24,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 in 163 msec 2024-11-14T07:16:24,512 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, REOPEN/MOVE; state=CLOSED, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=true 2024-11-14T07:16:24,663 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:24,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, REOPEN/MOVE because future has completed 2024-11-14T07:16:24,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:16:24,824 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,824 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:16:24,824 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,824 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:16:24,824 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,825 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,828 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,829 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:24,834 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fca7a5cd511ec9993090c4fc5431d4 columnFamilyName A 2024-11-14T07:16:24,836 DEBUG [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:24,837 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(327): Store=26fca7a5cd511ec9993090c4fc5431d4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:24,837 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,838 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:24,838 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fca7a5cd511ec9993090c4fc5431d4 columnFamilyName B 2024-11-14T07:16:24,839 DEBUG [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:24,839 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(327): Store=26fca7a5cd511ec9993090c4fc5431d4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:24,839 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,840 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:24,840 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fca7a5cd511ec9993090c4fc5431d4 columnFamilyName C 2024-11-14T07:16:24,840 DEBUG [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:24,841 INFO [StoreOpener-26fca7a5cd511ec9993090c4fc5431d4-1 {}] regionserver.HStore(327): Store=26fca7a5cd511ec9993090c4fc5431d4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:24,841 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,841 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,842 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,843 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,844 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,844 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,844 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:16:24,846 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,847 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 26fca7a5cd511ec9993090c4fc5431d4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63470895, jitterRate=-0.054209962487220764}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:16:24,847 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,848 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 26fca7a5cd511ec9993090c4fc5431d4: Running coprocessor pre-open hook at 1731568584825Writing region info on filesystem at 1731568584825Initializing all the Stores at 1731568584826 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568584826Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568584826Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568584826Cleaning up temporary data from old regions at 1731568584844 (+18 ms)Running coprocessor post-open hooks at 1731568584847 (+3 ms)Region opened successfully at 1731568584848 (+1 ms) 2024-11-14T07:16:24,849 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., pid=39, masterSystemTime=1731568584820 2024-11-14T07:16:24,851 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,851 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:24,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=OPEN, openSeqNum=5, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:24,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:16:24,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-14T07:16:24,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 in 187 msec 2024-11-14T07:16:24,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-14T07:16:24,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, REOPEN/MOVE in 515 msec 2024-11-14T07:16:24,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-14T07:16:24,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-14T07:16:24,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-11-14T07:16:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-14T07:16:24,871 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:24,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4921b08b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d7d22e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,882 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d385c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,885 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bbfc4e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739d6875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,889 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d63f034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d02de16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,894 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac82959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d263949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,899 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8ad2ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e2d8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a2829d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,905 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200a330f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c9db6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f795a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:24,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:24,912 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fc943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:24,914 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:24,916 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,916 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,920 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T07:16:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T07:16:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,928 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:24,929 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:24,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:24,932 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,932 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:24,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,942 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,953 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,977 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,979 DEBUG [Thread-592 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,979 DEBUG [Thread-592 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:24,988 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:24,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:16:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731568584964, max timestamp=1731568584995Num uniques 859; 2024-11-14T07:16:25,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-14T07:16:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568644988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731568584964, max timestamp=1731568584995Num uniques 859; 2024-11-14T07:16:25,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-14T07:16:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b8faff48c31a49738cbe9726ac3fedb7_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568584936/Put/seqid=0 2024-11-14T07:16:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21014, totalHeapSize=47976, min timestamp=1731568584964, max timestamp=1731568584995Num uniques 859; 2024-11-14T07:16:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=21014, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-14T07:16:25,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741939_1115 (size=12154) 2024-11-14T07:16:25,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:25,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b8faff48c31a49738cbe9726ac3fedb7_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b8faff48c31a49738cbe9726ac3fedb7_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:25,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9af5d02133484c4d9060bc599756f466, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T07:16:25,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9af5d02133484c4d9060bc599756f466 is 175, key is test_row_0/A:col10/1731568584936/Put/seqid=0 2024-11-14T07:16:25,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568645027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568645037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741940_1116 (size=30955) 2024-11-14T07:16:25,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9af5d02133484c4d9060bc599756f466 2024-11-14T07:16:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568645040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568645041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/95fea476c6c6416f8e3bb22874e71a4d is 50, key is test_row_0/B:col10/1731568584936/Put/seqid=0 2024-11-14T07:16:25,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741941_1117 (size=12001) 2024-11-14T07:16:25,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/95fea476c6c6416f8e3bb22874e71a4d 2024-11-14T07:16:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568645112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7b5f413f4dd64ca2ac9d1c443dfa298b is 50, key is test_row_0/C:col10/1731568584936/Put/seqid=0 2024-11-14T07:16:25,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741942_1118 (size=12001) 2024-11-14T07:16:25,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7b5f413f4dd64ca2ac9d1c443dfa298b 2024-11-14T07:16:25,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9af5d02133484c4d9060bc599756f466 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9af5d02133484c4d9060bc599756f466 2024-11-14T07:16:25,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9af5d02133484c4d9060bc599756f466, entries=150, sequenceid=15, filesize=30.2 K 2024-11-14T07:16:25,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/95fea476c6c6416f8e3bb22874e71a4d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/95fea476c6c6416f8e3bb22874e71a4d 2024-11-14T07:16:25,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568645152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568645152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/95fea476c6c6416f8e3bb22874e71a4d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:16:25,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7b5f413f4dd64ca2ac9d1c443dfa298b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7b5f413f4dd64ca2ac9d1c443dfa298b 2024-11-14T07:16:25,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7b5f413f4dd64ca2ac9d1c443dfa298b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:16:25,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 26fca7a5cd511ec9993090c4fc5431d4 in 248ms, sequenceid=15, compaction requested=false 2024-11-14T07:16:25,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:25,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=147.60 KB heapSize=396.70 KB 2024-11-14T07:16:25,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:25,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e75029505a5c4f0eb79cc331f1971136_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568585190/Put/seqid=0 2024-11-14T07:16:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T07:16:25,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741943_1119 (size=17034) 2024-11-14T07:16:25,252 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:25,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e75029505a5c4f0eb79cc331f1971136_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e75029505a5c4f0eb79cc331f1971136_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:25,266 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/6bd1e92e64724c929fe8d0c8bc945a33, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:25,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/6bd1e92e64724c929fe8d0c8bc945a33 is 175, key is test_row_0/A:col10/1731568585190/Put/seqid=0 2024-11-14T07:16:25,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568645303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568645310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568645323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741944_1120 (size=48139) 2024-11-14T07:16:25,346 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/6bd1e92e64724c929fe8d0c8bc945a33 2024-11-14T07:16:25,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/07d0afd057574843a67713995af3d155 is 50, key is test_row_0/B:col10/1731568585190/Put/seqid=0 2024-11-14T07:16:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568645372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568645382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:25,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:25,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741945_1121 (size=12001) 2024-11-14T07:16:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568645433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568645442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:25,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T07:16:25,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568645642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568645662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568645662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568645692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568645693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:25,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:25,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:25,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/07d0afd057574843a67713995af3d155 2024-11-14T07:16:25,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/b9733341dd294eb2b3503cbae6700eaf is 50, key is test_row_0/C:col10/1731568585190/Put/seqid=0 2024-11-14T07:16:25,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:25,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:25,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:25,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:25,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741946_1122 (size=12001) 2024-11-14T07:16:25,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/b9733341dd294eb2b3503cbae6700eaf 2024-11-14T07:16:25,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/6bd1e92e64724c929fe8d0c8bc945a33 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/6bd1e92e64724c929fe8d0c8bc945a33 2024-11-14T07:16:25,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/6bd1e92e64724c929fe8d0c8bc945a33, entries=250, sequenceid=48, filesize=47.0 K 2024-11-14T07:16:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/07d0afd057574843a67713995af3d155 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/07d0afd057574843a67713995af3d155 2024-11-14T07:16:25,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/07d0afd057574843a67713995af3d155, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T07:16:25,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/b9733341dd294eb2b3503cbae6700eaf as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/b9733341dd294eb2b3503cbae6700eaf 2024-11-14T07:16:25,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/b9733341dd294eb2b3503cbae6700eaf, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T07:16:25,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~396.66 KB/406176, currentSize=40.25 KB/41220 for 26fca7a5cd511ec9993090c4fc5431d4 in 718ms, sequenceid=48, compaction requested=false 2024-11-14T07:16:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:25,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:25,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111494631af17dae4b349c8ecf0c6900f7e0_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568585972/Put/seqid=0 2024-11-14T07:16:26,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:26,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741947_1123 (size=14594) 2024-11-14T07:16:26,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:26,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731568585993, max timestamp=1731568586039Num uniques 859; 2024-11-14T07:16:26,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111494631af17dae4b349c8ecf0c6900f7e0_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494631af17dae4b349c8ecf0c6900f7e0_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:26,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731568585993, max timestamp=1731568586039Num uniques 859; 2024-11-14T07:16:26,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T07:16:26,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T07:16:26,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731568585993, max timestamp=1731568586039Num uniques 859; 2024-11-14T07:16:26,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T07:16:26,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/12270f52b16f458b991c727654d1b4db, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:26,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/12270f52b16f458b991c727654d1b4db is 175, key is test_row_0/A:col10/1731568585972/Put/seqid=0 2024-11-14T07:16:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T07:16:26,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741948_1124 (size=39549) 2024-11-14T07:16:26,078 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/12270f52b16f458b991c727654d1b4db 2024-11-14T07:16:26,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/c80461dc07f340efba7d78d70109cbb0 is 50, key is test_row_0/B:col10/1731568585972/Put/seqid=0 2024-11-14T07:16:26,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568646111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568646117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741949_1125 (size=12001) 2024-11-14T07:16:26,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/c80461dc07f340efba7d78d70109cbb0 2024-11-14T07:16:26,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/8e1d3bbd36254fb99092362c565a7f9a is 50, key is test_row_0/C:col10/1731568585972/Put/seqid=0 2024-11-14T07:16:26,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568646153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:26,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:26,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:26,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:26,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741950_1126 (size=12001) 2024-11-14T07:16:26,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/8e1d3bbd36254fb99092362c565a7f9a 2024-11-14T07:16:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/12270f52b16f458b991c727654d1b4db as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/12270f52b16f458b991c727654d1b4db 2024-11-14T07:16:26,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/12270f52b16f458b991c727654d1b4db, entries=200, sequenceid=58, filesize=38.6 K 2024-11-14T07:16:26,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/c80461dc07f340efba7d78d70109cbb0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c80461dc07f340efba7d78d70109cbb0 2024-11-14T07:16:26,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568646202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568646202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c80461dc07f340efba7d78d70109cbb0, entries=150, sequenceid=58, filesize=11.7 K 2024-11-14T07:16:26,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/8e1d3bbd36254fb99092362c565a7f9a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/8e1d3bbd36254fb99092362c565a7f9a 2024-11-14T07:16:26,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568646222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568646232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/8e1d3bbd36254fb99092362c565a7f9a, entries=150, sequenceid=58, filesize=11.7 K 2024-11-14T07:16:26,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 26fca7a5cd511ec9993090c4fc5431d4 in 267ms, sequenceid=58, compaction requested=true 2024-11-14T07:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:26,240 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:16:26,240 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:26,243 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:26,243 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:26,243 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,243 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9af5d02133484c4d9060bc599756f466, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/6bd1e92e64724c929fe8d0c8bc945a33, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/12270f52b16f458b991c727654d1b4db] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=115.9 K 2024-11-14T07:16:26,243 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,244 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9af5d02133484c4d9060bc599756f466, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/6bd1e92e64724c929fe8d0c8bc945a33, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/12270f52b16f458b991c727654d1b4db] 2024-11-14T07:16:26,244 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:26,244 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:26,244 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,244 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/95fea476c6c6416f8e3bb22874e71a4d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/07d0afd057574843a67713995af3d155, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c80461dc07f340efba7d78d70109cbb0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.2 K 2024-11-14T07:16:26,245 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 95fea476c6c6416f8e3bb22874e71a4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731568584930 2024-11-14T07:16:26,245 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af5d02133484c4d9060bc599756f466, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731568584930 2024-11-14T07:16:26,246 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d0afd057574843a67713995af3d155, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731568585038 2024-11-14T07:16:26,246 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd1e92e64724c929fe8d0c8bc945a33, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731568585025 2024-11-14T07:16:26,246 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c80461dc07f340efba7d78d70109cbb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731568585228 2024-11-14T07:16:26,247 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12270f52b16f458b991c727654d1b4db, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731568585228 2024-11-14T07:16:26,265 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:26,271 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:26,272 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/8b28ddd826d64feaba9c6cbc49442d8c is 50, key is test_row_0/B:col10/1731568585972/Put/seqid=0 2024-11-14T07:16:26,278 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f52a50652bf54f4f8fcc9121d00e3f92_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:26,293 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f52a50652bf54f4f8fcc9121d00e3f92_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:26,293 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f52a50652bf54f4f8fcc9121d00e3f92_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:26,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741951_1127 (size=12104) 2024-11-14T07:16:26,318 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/8b28ddd826d64feaba9c6cbc49442d8c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8b28ddd826d64feaba9c6cbc49442d8c 2024-11-14T07:16:26,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T07:16:26,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-14T07:16:26,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:26,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:26,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:26,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:26,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:26,328 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into 8b28ddd826d64feaba9c6cbc49442d8c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:26,328 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:26,328 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=13, startTime=1731568586240; duration=0sec 2024-11-14T07:16:26,328 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:26,328 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:26,329 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:26,331 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:26,333 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:26,334 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:26,334 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7b5f413f4dd64ca2ac9d1c443dfa298b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/b9733341dd294eb2b3503cbae6700eaf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/8e1d3bbd36254fb99092362c565a7f9a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.2 K 2024-11-14T07:16:26,335 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b5f413f4dd64ca2ac9d1c443dfa298b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731568584930 2024-11-14T07:16:26,335 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting b9733341dd294eb2b3503cbae6700eaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731568585038 2024-11-14T07:16:26,336 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e1d3bbd36254fb99092362c565a7f9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731568585228 2024-11-14T07:16:26,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741952_1128 (size=4469) 2024-11-14T07:16:26,344 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:16:26,344 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#108 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:26,345 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a0306a14ea0d4754bdbc23e55c7fa3d6 is 175, key is test_row_0/A:col10/1731568585972/Put/seqid=0 2024-11-14T07:16:26,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ec27ededf0d4649a25ad1f275c4a6f0_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568586114/Put/seqid=0 2024-11-14T07:16:26,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741953_1129 (size=31058) 2024-11-14T07:16:26,401 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:26,402 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/4bd337a5723d4d179debc5f9870a1a26 is 50, key is test_row_0/C:col10/1731568585972/Put/seqid=0 2024-11-14T07:16:26,406 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a0306a14ea0d4754bdbc23e55c7fa3d6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0306a14ea0d4754bdbc23e55c7fa3d6 2024-11-14T07:16:26,414 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into a0306a14ea0d4754bdbc23e55c7fa3d6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:26,414 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:26,414 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=13, startTime=1731568586240; duration=0sec 2024-11-14T07:16:26,414 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:26,414 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:26,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741954_1130 (size=12154) 2024-11-14T07:16:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:26,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:26,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741955_1131 (size=12104) 2024-11-14T07:16:26,462 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/4bd337a5723d4d179debc5f9870a1a26 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4bd337a5723d4d179debc5f9870a1a26 2024-11-14T07:16:26,470 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into 4bd337a5723d4d179debc5f9870a1a26(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:26,471 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:26,471 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=13, startTime=1731568586240; duration=0sec 2024-11-14T07:16:26,471 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:26,471 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568646465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568646468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568646583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568646583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568646792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568646792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:26,829 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ec27ededf0d4649a25ad1f275c4a6f0_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ec27ededf0d4649a25ad1f275c4a6f0_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:26,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c481577310754948adfb103039480529, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c481577310754948adfb103039480529 is 175, key is test_row_0/A:col10/1731568586114/Put/seqid=0 2024-11-14T07:16:26,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741956_1132 (size=30955) 2024-11-14T07:16:26,881 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c481577310754948adfb103039480529 2024-11-14T07:16:26,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/ebe442317d5349039d3afbdbfc5af3eb is 50, key is test_row_0/B:col10/1731568586114/Put/seqid=0 2024-11-14T07:16:26,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741957_1133 (size=12001) 2024-11-14T07:16:26,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/ebe442317d5349039d3afbdbfc5af3eb 2024-11-14T07:16:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/4fb6c43a3d0d475697646e9f77c83c8c is 50, key is test_row_0/C:col10/1731568586114/Put/seqid=0 2024-11-14T07:16:27,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741958_1134 (size=12001) 2024-11-14T07:16:27,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/4fb6c43a3d0d475697646e9f77c83c8c 2024-11-14T07:16:27,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c481577310754948adfb103039480529 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c481577310754948adfb103039480529 2024-11-14T07:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T07:16:27,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c481577310754948adfb103039480529, entries=150, sequenceid=93, filesize=30.2 K 2024-11-14T07:16:27,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/ebe442317d5349039d3afbdbfc5af3eb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/ebe442317d5349039d3afbdbfc5af3eb 2024-11-14T07:16:27,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568647102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568647102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,108 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/ebe442317d5349039d3afbdbfc5af3eb, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T07:16:27,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/4fb6c43a3d0d475697646e9f77c83c8c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4fb6c43a3d0d475697646e9f77c83c8c 2024-11-14T07:16:27,125 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4fb6c43a3d0d475697646e9f77c83c8c, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T07:16:27,129 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 26fca7a5cd511ec9993090c4fc5431d4 in 805ms, sequenceid=93, compaction requested=false 2024-11-14T07:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T07:16:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T07:16:27,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T07:16:27,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2070 sec 2024-11-14T07:16:27,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2160 sec 2024-11-14T07:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:27,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:16:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:27,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:27,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:27,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:27,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:27,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143279e02969714954a64e5e4501fd979f_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568586458/Put/seqid=0 2024-11-14T07:16:27,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741959_1135 (size=14594) 2024-11-14T07:16:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568647292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568587239, max timestamp=1731568587304Num uniques 859; 2024-11-14T07:16:27,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568587239, max timestamp=1731568587304Num uniques 859; 2024-11-14T07:16:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:16:27,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:16:27,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568587239, max timestamp=1731568587304Num uniques 859; 2024-11-14T07:16:27,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:16:27,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568647410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568647412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568647414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568647522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568647541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568647612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568647616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568647643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:27,670 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143279e02969714954a64e5e4501fd979f_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143279e02969714954a64e5e4501fd979f_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:27,671 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c488d373b1054c47ac4178e5eb71431a, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:27,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c488d373b1054c47ac4178e5eb71431a is 175, key is test_row_0/A:col10/1731568586458/Put/seqid=0 2024-11-14T07:16:27,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741960_1136 (size=39549) 2024-11-14T07:16:27,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c488d373b1054c47ac4178e5eb71431a 2024-11-14T07:16:27,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/b662bf506fcf42c9b8b45777cdd5b599 is 50, key is test_row_0/B:col10/1731568586458/Put/seqid=0 2024-11-14T07:16:27,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568647743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741961_1137 (size=12001) 2024-11-14T07:16:27,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568647753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:27,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568647962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568648063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568648082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/b662bf506fcf42c9b8b45777cdd5b599 2024-11-14T07:16:28,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7bd8628162bf4a358468f9f9682201f3 is 50, key is test_row_0/C:col10/1731568586458/Put/seqid=0 2024-11-14T07:16:28,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741962_1138 (size=12001) 2024-11-14T07:16:28,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7bd8628162bf4a358468f9f9682201f3 2024-11-14T07:16:28,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/c488d373b1054c47ac4178e5eb71431a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c488d373b1054c47ac4178e5eb71431a 2024-11-14T07:16:28,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c488d373b1054c47ac4178e5eb71431a, entries=200, sequenceid=107, filesize=38.6 K 2024-11-14T07:16:28,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/b662bf506fcf42c9b8b45777cdd5b599 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b662bf506fcf42c9b8b45777cdd5b599 2024-11-14T07:16:28,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b662bf506fcf42c9b8b45777cdd5b599, entries=150, sequenceid=107, filesize=11.7 K 2024-11-14T07:16:28,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7bd8628162bf4a358468f9f9682201f3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7bd8628162bf4a358468f9f9682201f3 2024-11-14T07:16:28,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7bd8628162bf4a358468f9f9682201f3, entries=150, sequenceid=107, filesize=11.7 K 2024-11-14T07:16:28,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 26fca7a5cd511ec9993090c4fc5431d4 in 1089ms, sequenceid=107, compaction requested=true 2024-11-14T07:16:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:28,268 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:16:28,268 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:28,271 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:28,272 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:28,272 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:28,272 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0306a14ea0d4754bdbc23e55c7fa3d6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c481577310754948adfb103039480529, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c488d373b1054c47ac4178e5eb71431a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=99.2 K 2024-11-14T07:16:28,272 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:28,272 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:28,272 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:28,272 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0306a14ea0d4754bdbc23e55c7fa3d6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c481577310754948adfb103039480529, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c488d373b1054c47ac4178e5eb71431a] 2024-11-14T07:16:28,272 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:28,272 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8b28ddd826d64feaba9c6cbc49442d8c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/ebe442317d5349039d3afbdbfc5af3eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b662bf506fcf42c9b8b45777cdd5b599] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.3 K 2024-11-14T07:16:28,273 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0306a14ea0d4754bdbc23e55c7fa3d6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731568585228 2024-11-14T07:16:28,273 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b28ddd826d64feaba9c6cbc49442d8c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731568585228 2024-11-14T07:16:28,274 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting c481577310754948adfb103039480529, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731568586070 2024-11-14T07:16:28,274 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe442317d5349039d3afbdbfc5af3eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731568586070 2024-11-14T07:16:28,274 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting c488d373b1054c47ac4178e5eb71431a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731568586439 2024-11-14T07:16:28,274 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting b662bf506fcf42c9b8b45777cdd5b599, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731568586458 2024-11-14T07:16:28,305 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:28,315 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:28,316 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/6e0db8c62c094198acf8abffe64890d5 is 50, key is test_row_0/B:col10/1731568586458/Put/seqid=0 2024-11-14T07:16:28,327 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d04109fee49c4e289f23c417ec9794a4_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:28,330 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d04109fee49c4e289f23c417ec9794a4_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:28,330 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d04109fee49c4e289f23c417ec9794a4_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:28,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741963_1139 (size=12207) 2024-11-14T07:16:28,379 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/6e0db8c62c094198acf8abffe64890d5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/6e0db8c62c094198acf8abffe64890d5 2024-11-14T07:16:28,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741964_1140 (size=4469) 2024-11-14T07:16:28,388 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#117 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:28,388 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/eec383c16b534d12865ed0a93e75aef1 is 175, key is test_row_0/A:col10/1731568586458/Put/seqid=0 2024-11-14T07:16:28,390 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into 6e0db8c62c094198acf8abffe64890d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:28,390 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:28,390 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=13, startTime=1731568588268; duration=0sec 2024-11-14T07:16:28,390 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:28,390 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:28,390 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:28,393 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:28,393 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:28,393 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:28,393 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4bd337a5723d4d179debc5f9870a1a26, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4fb6c43a3d0d475697646e9f77c83c8c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7bd8628162bf4a358468f9f9682201f3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.3 K 2024-11-14T07:16:28,394 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd337a5723d4d179debc5f9870a1a26, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731568585228 2024-11-14T07:16:28,395 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb6c43a3d0d475697646e9f77c83c8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731568586070 2024-11-14T07:16:28,396 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd8628162bf4a358468f9f9682201f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731568586458 2024-11-14T07:16:28,410 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:28,411 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/fd8f363c51514ab6934c11c1bbe53bb7 is 50, key is test_row_0/C:col10/1731568586458/Put/seqid=0 2024-11-14T07:16:28,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741965_1141 (size=31161) 2024-11-14T07:16:28,422 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/eec383c16b534d12865ed0a93e75aef1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eec383c16b534d12865ed0a93e75aef1 2024-11-14T07:16:28,433 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into eec383c16b534d12865ed0a93e75aef1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:28,433 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:28,433 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=13, startTime=1731568588268; duration=0sec 2024-11-14T07:16:28,433 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:28,433 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:28,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741966_1142 (size=12207) 2024-11-14T07:16:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:28,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T07:16:28,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:28,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d252e37077c44917bbabbb987a05fcfd_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568588486/Put/seqid=0 2024-11-14T07:16:28,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741967_1143 (size=14794) 2024-11-14T07:16:28,535 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:28,545 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d252e37077c44917bbabbb987a05fcfd_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d252e37077c44917bbabbb987a05fcfd_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:28,546 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/eae70adf754948838ff27d57727172e0, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:28,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568648539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/eae70adf754948838ff27d57727172e0 is 175, key is test_row_0/A:col10/1731568588486/Put/seqid=0 2024-11-14T07:16:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741968_1144 (size=39749) 2024-11-14T07:16:28,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/eae70adf754948838ff27d57727172e0 2024-11-14T07:16:28,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/8f3d4345e56e42e8b7d316a2378c54ad is 50, key is test_row_0/B:col10/1731568588486/Put/seqid=0 2024-11-14T07:16:28,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568648583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741969_1145 (size=12151) 2024-11-14T07:16:28,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/8f3d4345e56e42e8b7d316a2378c54ad 2024-11-14T07:16:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568648633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,635 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568648633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,638 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/567b07f6889d4b6f8f9d01e8278bbc70 is 50, key is test_row_0/C:col10/1731568588486/Put/seqid=0 2024-11-14T07:16:28,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568648652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741970_1146 (size=12151) 2024-11-14T07:16:28,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568648862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:28,900 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/fd8f363c51514ab6934c11c1bbe53bb7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/fd8f363c51514ab6934c11c1bbe53bb7 2024-11-14T07:16:28,915 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into fd8f363c51514ab6934c11c1bbe53bb7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:28,915 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:28,916 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=13, startTime=1731568588268; duration=0sec 2024-11-14T07:16:28,916 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:28,916 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T07:16:29,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/567b07f6889d4b6f8f9d01e8278bbc70 2024-11-14T07:16:29,104 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:29,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T07:16:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:16:29,109 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:29,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:29,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:29,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/eae70adf754948838ff27d57727172e0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eae70adf754948838ff27d57727172e0 2024-11-14T07:16:29,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eae70adf754948838ff27d57727172e0, entries=200, sequenceid=146, filesize=38.8 K 2024-11-14T07:16:29,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/8f3d4345e56e42e8b7d316a2378c54ad as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8f3d4345e56e42e8b7d316a2378c54ad 2024-11-14T07:16:29,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8f3d4345e56e42e8b7d316a2378c54ad, entries=150, sequenceid=146, filesize=11.9 K 2024-11-14T07:16:29,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/567b07f6889d4b6f8f9d01e8278bbc70 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/567b07f6889d4b6f8f9d01e8278bbc70 2024-11-14T07:16:29,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/567b07f6889d4b6f8f9d01e8278bbc70, entries=150, sequenceid=146, filesize=11.9 K 2024-11-14T07:16:29,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 26fca7a5cd511ec9993090c4fc5431d4 in 687ms, sequenceid=146, compaction requested=false 2024-11-14T07:16:29,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:29,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:16:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:16:29,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3f98aa3db7640c897f521e0ff625f49_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568588503/Put/seqid=0 2024-11-14T07:16:29,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741971_1147 (size=14794) 2024-11-14T07:16:29,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:29,300 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3f98aa3db7640c897f521e0ff625f49_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3f98aa3db7640c897f521e0ff625f49_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:29,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a888cb4036bf46698d3d1facfabf8272, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:29,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a888cb4036bf46698d3d1facfabf8272 is 175, key is test_row_0/A:col10/1731568588503/Put/seqid=0 2024-11-14T07:16:29,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741972_1148 (size=39749) 2024-11-14T07:16:29,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568589310, max timestamp=1731568589387Num uniques 859; 2024-11-14T07:16:29,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:29,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568589310, max timestamp=1731568589387Num uniques 859; 2024-11-14T07:16:29,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:29,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568589310, max timestamp=1731568589387Num uniques 859; 2024-11-14T07:16:29,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:16:29,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:29,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568649553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:29,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568649603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:29,610 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568649633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:29,644 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568649662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:16:29,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a888cb4036bf46698d3d1facfabf8272 2024-11-14T07:16:29,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/e6aec13dafb44279baeab46407491269 is 50, key is test_row_0/B:col10/1731568588503/Put/seqid=0 2024-11-14T07:16:29,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741973_1149 (size=12151) 2024-11-14T07:16:29,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568649872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:29,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:29,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:29,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:29,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:30,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:30,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/e6aec13dafb44279baeab46407491269 2024-11-14T07:16:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568650182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:30,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:30,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7eb01b50a0964b2aaef1c89454a3f1f4 is 50, key is test_row_0/C:col10/1731568588503/Put/seqid=0 2024-11-14T07:16:30,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741974_1150 (size=12151) 2024-11-14T07:16:30,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7eb01b50a0964b2aaef1c89454a3f1f4 2024-11-14T07:16:30,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:30,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:30,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:30,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:30,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a888cb4036bf46698d3d1facfabf8272 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a888cb4036bf46698d3d1facfabf8272 2024-11-14T07:16:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:16:30,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a888cb4036bf46698d3d1facfabf8272, entries=200, sequenceid=158, filesize=38.8 K 2024-11-14T07:16:30,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/e6aec13dafb44279baeab46407491269 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e6aec13dafb44279baeab46407491269 2024-11-14T07:16:30,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e6aec13dafb44279baeab46407491269, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T07:16:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/7eb01b50a0964b2aaef1c89454a3f1f4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7eb01b50a0964b2aaef1c89454a3f1f4 2024-11-14T07:16:30,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7eb01b50a0964b2aaef1c89454a3f1f4, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T07:16:30,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 26fca7a5cd511ec9993090c4fc5431d4 in 1070ms, sequenceid=158, compaction requested=true 2024-11-14T07:16:30,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:30,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:30,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:30,262 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:30,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:30,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:30,262 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:30,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:30,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:30,267 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:30,267 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:30,267 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,267 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eec383c16b534d12865ed0a93e75aef1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eae70adf754948838ff27d57727172e0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a888cb4036bf46698d3d1facfabf8272] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=108.1 K 2024-11-14T07:16:30,267 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,267 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eec383c16b534d12865ed0a93e75aef1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eae70adf754948838ff27d57727172e0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a888cb4036bf46698d3d1facfabf8272] 2024-11-14T07:16:30,268 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:30,268 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:30,268 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,268 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/6e0db8c62c094198acf8abffe64890d5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8f3d4345e56e42e8b7d316a2378c54ad, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e6aec13dafb44279baeab46407491269] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.7 K 2024-11-14T07:16:30,268 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec383c16b534d12865ed0a93e75aef1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731568586458 2024-11-14T07:16:30,270 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0db8c62c094198acf8abffe64890d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731568586458 2024-11-14T07:16:30,270 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae70adf754948838ff27d57727172e0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568587362 2024-11-14T07:16:30,271 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a888cb4036bf46698d3d1facfabf8272, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731568588503 2024-11-14T07:16:30,271 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f3d4345e56e42e8b7d316a2378c54ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568587405 2024-11-14T07:16:30,272 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e6aec13dafb44279baeab46407491269, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731568588503 2024-11-14T07:16:30,293 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:30,302 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:30,303 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/f44b6aeaf92a4515acf37c6556add5bb is 50, key is test_row_0/B:col10/1731568588503/Put/seqid=0 2024-11-14T07:16:30,313 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114fbbecb377c584812b47e0e86c8537d97_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:30,316 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114fbbecb377c584812b47e0e86c8537d97_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:30,316 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fbbecb377c584812b47e0e86c8537d97_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:30,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741975_1151 (size=12459) 2024-11-14T07:16:30,365 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/f44b6aeaf92a4515acf37c6556add5bb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f44b6aeaf92a4515acf37c6556add5bb 2024-11-14T07:16:30,374 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into f44b6aeaf92a4515acf37c6556add5bb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:30,374 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:30,374 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=13, startTime=1731568590262; duration=0sec 2024-11-14T07:16:30,375 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:30,375 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:30,375 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:30,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:16:30,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,376 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T07:16:30,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:30,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:30,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:30,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:30,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:30,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:30,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741976_1152 (size=4469) 2024-11-14T07:16:30,378 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:30,380 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:30,380 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:30,380 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/fd8f363c51514ab6934c11c1bbe53bb7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/567b07f6889d4b6f8f9d01e8278bbc70, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7eb01b50a0964b2aaef1c89454a3f1f4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.7 K 2024-11-14T07:16:30,381 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#126 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:30,381 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting fd8f363c51514ab6934c11c1bbe53bb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731568586458 2024-11-14T07:16:30,381 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/2c123ec839874d1ca994e0837e39adc2 is 175, key is test_row_0/A:col10/1731568588503/Put/seqid=0 2024-11-14T07:16:30,383 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 567b07f6889d4b6f8f9d01e8278bbc70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568587405 2024-11-14T07:16:30,384 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eb01b50a0964b2aaef1c89454a3f1f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731568588503 2024-11-14T07:16:30,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741977_1153 (size=31413) 2024-11-14T07:16:30,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114476ffc8a5cf34c4dbc43914c0191eafd_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568589477/Put/seqid=0 2024-11-14T07:16:30,442 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:30,442 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/c1f7da0cac0a4296b8468af20c7ea0c5 is 50, key is test_row_0/C:col10/1731568588503/Put/seqid=0 2024-11-14T07:16:30,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741978_1154 (size=12304) 2024-11-14T07:16:30,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:30,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741979_1155 (size=12459) 2024-11-14T07:16:30,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114476ffc8a5cf34c4dbc43914c0191eafd_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114476ffc8a5cf34c4dbc43914c0191eafd_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:30,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b12c47a08a0a4775b62ab69cc09bf8fb, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:30,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b12c47a08a0a4775b62ab69cc09bf8fb is 175, key is test_row_0/A:col10/1731568589477/Put/seqid=0 2024-11-14T07:16:30,505 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/c1f7da0cac0a4296b8468af20c7ea0c5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1f7da0cac0a4296b8468af20c7ea0c5 2024-11-14T07:16:30,515 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into c1f7da0cac0a4296b8468af20c7ea0c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:30,515 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:30,515 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=13, startTime=1731568590263; duration=0sec 2024-11-14T07:16:30,515 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:30,515 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:30,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741980_1156 (size=31105) 2024-11-14T07:16:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:30,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:30,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568650694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:30,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568650701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:30,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568650704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:30,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568650813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:30,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568650813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:30,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568650822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:30,837 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/2c123ec839874d1ca994e0837e39adc2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/2c123ec839874d1ca994e0837e39adc2 2024-11-14T07:16:30,849 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into 2c123ec839874d1ca994e0837e39adc2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:30,849 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:30,849 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=13, startTime=1731568590262; duration=0sec 2024-11-14T07:16:30,850 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:30,850 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:30,935 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b12c47a08a0a4775b62ab69cc09bf8fb 2024-11-14T07:16:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/bb49dd8c9b4d4070bfc268258300765f is 50, key is test_row_0/B:col10/1731568589477/Put/seqid=0 2024-11-14T07:16:30,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741981_1157 (size=12151) 2024-11-14T07:16:31,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568651022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568651022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568651032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:16:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568651331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568651331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568651342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/bb49dd8c9b4d4070bfc268258300765f 2024-11-14T07:16:31,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/2f6d22ac3f88449e99d4be31eeb5c84d is 50, key is test_row_0/C:col10/1731568589477/Put/seqid=0 2024-11-14T07:16:31,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741982_1158 (size=12151) 2024-11-14T07:16:31,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568651632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,636 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568651672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,676 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,801 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/2f6d22ac3f88449e99d4be31eeb5c84d 2024-11-14T07:16:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b12c47a08a0a4775b62ab69cc09bf8fb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b12c47a08a0a4775b62ab69cc09bf8fb 2024-11-14T07:16:31,825 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b12c47a08a0a4775b62ab69cc09bf8fb, entries=150, sequenceid=194, filesize=30.4 K 2024-11-14T07:16:31,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/bb49dd8c9b4d4070bfc268258300765f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/bb49dd8c9b4d4070bfc268258300765f 2024-11-14T07:16:31,848 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/bb49dd8c9b4d4070bfc268258300765f, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T07:16:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/2f6d22ac3f88449e99d4be31eeb5c84d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/2f6d22ac3f88449e99d4be31eeb5c84d 2024-11-14T07:16:31,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568651842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568651843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:31,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/2f6d22ac3f88449e99d4be31eeb5c84d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T07:16:31,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 26fca7a5cd511ec9993090c4fc5431d4 in 1482ms, sequenceid=194, compaction requested=false 2024-11-14T07:16:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T07:16:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T07:16:31,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T07:16:31,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7490 sec 2024-11-14T07:16:31,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.7750 sec 2024-11-14T07:16:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:31,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:16:31,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:31,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:31,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:31,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:31,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:31,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:31,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ab0a3cebc22e454cbfa6115a568c5048_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568590680/Put/seqid=0 2024-11-14T07:16:31,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741983_1159 (size=14794) 2024-11-14T07:16:31,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:31,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ab0a3cebc22e454cbfa6115a568c5048_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ab0a3cebc22e454cbfa6115a568c5048_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:31,980 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ac97be209fdc402eb239770d2c10505b, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:31,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ac97be209fdc402eb239770d2c10505b is 175, key is test_row_0/A:col10/1731568590680/Put/seqid=0 2024-11-14T07:16:32,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741984_1160 (size=39749) 2024-11-14T07:16:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568652076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:32,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568652192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:32,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568652402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:32,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ac97be209fdc402eb239770d2c10505b 2024-11-14T07:16:32,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/e9c24141e81d40fcb52e4dde4a64e07c is 50, key is test_row_0/B:col10/1731568590680/Put/seqid=0 2024-11-14T07:16:32,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741985_1161 (size=12151) 2024-11-14T07:16:32,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568652712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:32,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/e9c24141e81d40fcb52e4dde4a64e07c 2024-11-14T07:16:32,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/41e1e08060ad475aa5b90eaa12421c67 is 50, key is test_row_0/C:col10/1731568590680/Put/seqid=0 2024-11-14T07:16:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741986_1162 (size=12151) 2024-11-14T07:16:32,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568652872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:32,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568652872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:32,874 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:32,874 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:33,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568653222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:33,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/41e1e08060ad475aa5b90eaa12421c67 2024-11-14T07:16:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ac97be209fdc402eb239770d2c10505b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ac97be209fdc402eb239770d2c10505b 2024-11-14T07:16:33,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ac97be209fdc402eb239770d2c10505b, entries=200, sequenceid=208, filesize=38.8 K 2024-11-14T07:16:33,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/e9c24141e81d40fcb52e4dde4a64e07c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e9c24141e81d40fcb52e4dde4a64e07c 2024-11-14T07:16:33,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e9c24141e81d40fcb52e4dde4a64e07c, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T07:16:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:16:33,272 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:33,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T07:16:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:16:33,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/41e1e08060ad475aa5b90eaa12421c67 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/41e1e08060ad475aa5b90eaa12421c67 2024-11-14T07:16:33,277 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:33,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:33,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:33,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/41e1e08060ad475aa5b90eaa12421c67, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T07:16:33,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 26fca7a5cd511ec9993090c4fc5431d4 in 1396ms, sequenceid=208, compaction requested=true 2024-11-14T07:16:33,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:33,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:33,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:33,288 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:33,288 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:33,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:33,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:33,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:33,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:33,290 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:33,290 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:33,290 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:33,290 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:33,290 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:33,290 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:33,290 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f44b6aeaf92a4515acf37c6556add5bb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/bb49dd8c9b4d4070bfc268258300765f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e9c24141e81d40fcb52e4dde4a64e07c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.9 K 2024-11-14T07:16:33,290 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/2c123ec839874d1ca994e0837e39adc2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b12c47a08a0a4775b62ab69cc09bf8fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ac97be209fdc402eb239770d2c10505b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=99.9 K 2024-11-14T07:16:33,291 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:33,291 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/2c123ec839874d1ca994e0837e39adc2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b12c47a08a0a4775b62ab69cc09bf8fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ac97be209fdc402eb239770d2c10505b] 2024-11-14T07:16:33,291 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f44b6aeaf92a4515acf37c6556add5bb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731568588503 2024-11-14T07:16:33,291 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c123ec839874d1ca994e0837e39adc2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731568588503 2024-11-14T07:16:33,292 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12c47a08a0a4775b62ab69cc09bf8fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731568589477 2024-11-14T07:16:33,292 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting bb49dd8c9b4d4070bfc268258300765f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731568589477 2024-11-14T07:16:33,292 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac97be209fdc402eb239770d2c10505b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731568590680 2024-11-14T07:16:33,293 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c24141e81d40fcb52e4dde4a64e07c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731568590680 2024-11-14T07:16:33,321 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:33,334 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:33,335 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/0a3d3298a84e4e6c89350b1e1d655934 is 50, key is test_row_0/B:col10/1731568590680/Put/seqid=0 2024-11-14T07:16:33,337 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111475bb125048464485a9bffe56e74d3a81_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:33,339 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111475bb125048464485a9bffe56e74d3a81_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:33,339 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111475bb125048464485a9bffe56e74d3a81_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741988_1164 (size=4469) 2024-11-14T07:16:33,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741987_1163 (size=12561) 2024-11-14T07:16:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:16:33,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:33,434 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:16:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c39510b7e6ab40da851097075293499f_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568592063/Put/seqid=0 2024-11-14T07:16:33,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741989_1165 (size=12304) 2024-11-14T07:16:33,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:33,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c39510b7e6ab40da851097075293499f_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c39510b7e6ab40da851097075293499f_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9feabc319d1543a7a3aed94b836dfb0e, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:33,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9feabc319d1543a7a3aed94b836dfb0e is 175, key is test_row_0/A:col10/1731568592063/Put/seqid=0 2024-11-14T07:16:33,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741990_1166 (size=31105) 2024-11-14T07:16:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:16:33,735 INFO [master/c7aa6e2790b3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T07:16:33,735 INFO [master/c7aa6e2790b3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T07:16:33,753 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#135 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:33,753 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/35148dca98e7403f9cdf8daa2fc73e65 is 175, key is test_row_0/A:col10/1731568590680/Put/seqid=0 2024-11-14T07:16:33,766 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/0a3d3298a84e4e6c89350b1e1d655934 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/0a3d3298a84e4e6c89350b1e1d655934 2024-11-14T07:16:33,780 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into 0a3d3298a84e4e6c89350b1e1d655934(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:33,780 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:33,780 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=13, startTime=1731568593288; duration=0sec 2024-11-14T07:16:33,780 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:33,780 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:33,780 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:33,782 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:33,782 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:33,782 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:33,782 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1f7da0cac0a4296b8468af20c7ea0c5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/2f6d22ac3f88449e99d4be31eeb5c84d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/41e1e08060ad475aa5b90eaa12421c67] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=35.9 K 2024-11-14T07:16:33,783 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f7da0cac0a4296b8468af20c7ea0c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731568588503 2024-11-14T07:16:33,783 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6d22ac3f88449e99d4be31eeb5c84d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731568589477 2024-11-14T07:16:33,784 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 41e1e08060ad475aa5b90eaa12421c67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731568590680 2024-11-14T07:16:33,857 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:33,857 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/c1a9c5186aa1492c8079fe6c1b8f1068 is 50, key is test_row_0/C:col10/1731568590680/Put/seqid=0 2024-11-14T07:16:33,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741991_1167 (size=31515) 2024-11-14T07:16:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:16:33,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741992_1168 (size=12561) 2024-11-14T07:16:33,941 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9feabc319d1543a7a3aed94b836dfb0e 2024-11-14T07:16:33,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/fd131b6b17db446f83665b367f3a7ae1 is 50, key is test_row_0/B:col10/1731568592063/Put/seqid=0 2024-11-14T07:16:34,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741993_1169 (size=12151) 2024-11-14T07:16:34,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/fd131b6b17db446f83665b367f3a7ae1 2024-11-14T07:16:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/5f8c2b0661d64ae5b9d950c379c3d7a9 is 50, key is test_row_0/C:col10/1731568592063/Put/seqid=0 2024-11-14T07:16:34,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741994_1170 (size=12151) 2024-11-14T07:16:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:34,265 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/35148dca98e7403f9cdf8daa2fc73e65 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/35148dca98e7403f9cdf8daa2fc73e65 2024-11-14T07:16:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:34,285 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into 35148dca98e7403f9cdf8daa2fc73e65(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:34,285 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568654285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:34,285 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=13, startTime=1731568593288; duration=0sec 2024-11-14T07:16:34,286 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:34,286 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:34,326 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/c1a9c5186aa1492c8079fe6c1b8f1068 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1a9c5186aa1492c8079fe6c1b8f1068 2024-11-14T07:16:34,334 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into c1a9c5186aa1492c8079fe6c1b8f1068(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:34,334 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:34,335 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=13, startTime=1731568593288; duration=0sec 2024-11-14T07:16:34,335 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:34,335 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568654392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:34,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:16:34,503 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/5f8c2b0661d64ae5b9d950c379c3d7a9 2024-11-14T07:16:34,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9feabc319d1543a7a3aed94b836dfb0e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9feabc319d1543a7a3aed94b836dfb0e 2024-11-14T07:16:34,524 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9feabc319d1543a7a3aed94b836dfb0e, entries=150, sequenceid=228, filesize=30.4 K 2024-11-14T07:16:34,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/fd131b6b17db446f83665b367f3a7ae1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/fd131b6b17db446f83665b367f3a7ae1 2024-11-14T07:16:34,532 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/fd131b6b17db446f83665b367f3a7ae1, entries=150, sequenceid=228, filesize=11.9 K 2024-11-14T07:16:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/5f8c2b0661d64ae5b9d950c379c3d7a9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5f8c2b0661d64ae5b9d950c379c3d7a9 2024-11-14T07:16:34,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5f8c2b0661d64ae5b9d950c379c3d7a9, entries=150, sequenceid=228, filesize=11.9 K 2024-11-14T07:16:34,549 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 26fca7a5cd511ec9993090c4fc5431d4 in 1114ms, sequenceid=228, compaction requested=false 2024-11-14T07:16:34,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:34,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:34,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T07:16:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T07:16:34,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T07:16:34,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-11-14T07:16:34,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2900 sec 2024-11-14T07:16:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:34,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:16:34,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:34,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:34,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141188c96c9279483fbce744cf49cf88be_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568594602/Put/seqid=0 2024-11-14T07:16:34,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741995_1171 (size=12304) 2024-11-14T07:16:34,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568654684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:34,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568654792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:34,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568654881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:34,883 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:34,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568654892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:34,897 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:35,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568655002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:35,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:35,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141188c96c9279483fbce744cf49cf88be_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141188c96c9279483fbce744cf49cf88be_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:35,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/0ddbacd254f44a6fb6e3bec425f870a2, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:35,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/0ddbacd254f44a6fb6e3bec425f870a2 is 175, key is test_row_0/A:col10/1731568594602/Put/seqid=0 2024-11-14T07:16:35,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741996_1172 (size=31105) 2024-11-14T07:16:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568655312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:16:35,432 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:35,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:35,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/0ddbacd254f44a6fb6e3bec425f870a2 2024-11-14T07:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T07:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:16:35,434 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:35,435 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:35,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:35,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/f70461ce012b40c9bbd72f4449ad371c is 50, key is test_row_0/B:col10/1731568594602/Put/seqid=0 2024-11-14T07:16:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741997_1173 (size=12151) 2024-11-14T07:16:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:16:35,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:16:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:35,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46266 deadline: 1731568655662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:35,664 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:35,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46218 deadline: 1731568655701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:35,705 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8291 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:35,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:16:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:35,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:16:35,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568655822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:35,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/f70461ce012b40c9bbd72f4449ad371c 2024-11-14T07:16:35,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/f69cc867e7ef41d3a724c54d3fe25a02 is 50, key is test_row_0/C:col10/1731568594602/Put/seqid=0 2024-11-14T07:16:35,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741998_1174 (size=12151) 2024-11-14T07:16:35,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:16:35,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:35,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:35,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:35,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:36,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:16:36,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:36,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:36,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:16:36,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:16:36,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:36,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:36,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:36,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/f69cc867e7ef41d3a724c54d3fe25a02 2024-11-14T07:16:36,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/0ddbacd254f44a6fb6e3bec425f870a2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/0ddbacd254f44a6fb6e3bec425f870a2 2024-11-14T07:16:36,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/0ddbacd254f44a6fb6e3bec425f870a2, entries=150, sequenceid=244, filesize=30.4 K 2024-11-14T07:16:36,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/f70461ce012b40c9bbd72f4449ad371c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f70461ce012b40c9bbd72f4449ad371c 2024-11-14T07:16:36,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f70461ce012b40c9bbd72f4449ad371c, entries=150, sequenceid=244, filesize=11.9 K 2024-11-14T07:16:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/f69cc867e7ef41d3a724c54d3fe25a02 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f69cc867e7ef41d3a724c54d3fe25a02 2024-11-14T07:16:36,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f69cc867e7ef41d3a724c54d3fe25a02, entries=150, sequenceid=244, filesize=11.9 K 2024-11-14T07:16:36,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 26fca7a5cd511ec9993090c4fc5431d4 in 1684ms, sequenceid=244, compaction requested=true 2024-11-14T07:16:36,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:36,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:36,289 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:36,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:36,289 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:36,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:36,290 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:36,290 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:36,290 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,290 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/0a3d3298a84e4e6c89350b1e1d655934, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/fd131b6b17db446f83665b367f3a7ae1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f70461ce012b40c9bbd72f4449ad371c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=36.0 K 2024-11-14T07:16:36,291 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3d3298a84e4e6c89350b1e1d655934, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731568590680 2024-11-14T07:16:36,291 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:36,291 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:36,291 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,291 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/35148dca98e7403f9cdf8daa2fc73e65, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9feabc319d1543a7a3aed94b836dfb0e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/0ddbacd254f44a6fb6e3bec425f870a2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=91.5 K 2024-11-14T07:16:36,291 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,291 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/35148dca98e7403f9cdf8daa2fc73e65, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9feabc319d1543a7a3aed94b836dfb0e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/0ddbacd254f44a6fb6e3bec425f870a2] 2024-11-14T07:16:36,291 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting fd131b6b17db446f83665b367f3a7ae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731568592061 2024-11-14T07:16:36,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:36,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:36,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:36,293 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35148dca98e7403f9cdf8daa2fc73e65, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731568590680 2024-11-14T07:16:36,293 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f70461ce012b40c9bbd72f4449ad371c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731568594266 2024-11-14T07:16:36,293 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9feabc319d1543a7a3aed94b836dfb0e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731568592061 2024-11-14T07:16:36,294 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ddbacd254f44a6fb6e3bec425f870a2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731568594266 2024-11-14T07:16:36,304 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:36,304 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:36,304 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/c4fa755565f449d7bf00513882e91064 is 50, key is test_row_0/B:col10/1731568594602/Put/seqid=0 2024-11-14T07:16:36,323 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e7b306bec1aa4dda9c2a36a37042afcd_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:36,325 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e7b306bec1aa4dda9c2a36a37042afcd_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:36,325 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e7b306bec1aa4dda9c2a36a37042afcd_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:36,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741999_1175 (size=12663) 2024-11-14T07:16:36,351 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/c4fa755565f449d7bf00513882e91064 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c4fa755565f449d7bf00513882e91064 2024-11-14T07:16:36,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:16:36,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:16:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:36,364 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into c4fa755565f449d7bf00513882e91064(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:36,364 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:36,364 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=13, startTime=1731568596289; duration=0sec 2024-11-14T07:16:36,364 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:36,364 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:36,364 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142acce97db4684f32b6acf32d6a03789c_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568594672/Put/seqid=0 2024-11-14T07:16:36,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742000_1176 (size=4469) 2024-11-14T07:16:36,367 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#145 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:36,368 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5bf2d0523ade4619ab596355b3d8e40a is 175, key is test_row_0/A:col10/1731568594602/Put/seqid=0 2024-11-14T07:16:36,368 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:36,368 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:36,368 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:36,369 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1a9c5186aa1492c8079fe6c1b8f1068, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5f8c2b0661d64ae5b9d950c379c3d7a9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f69cc867e7ef41d3a724c54d3fe25a02] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=36.0 K 2024-11-14T07:16:36,370 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a9c5186aa1492c8079fe6c1b8f1068, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731568590680 2024-11-14T07:16:36,370 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8c2b0661d64ae5b9d950c379c3d7a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731568592061 2024-11-14T07:16:36,372 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f69cc867e7ef41d3a724c54d3fe25a02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731568594266 2024-11-14T07:16:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742001_1177 (size=12404) 2024-11-14T07:16:36,404 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:36,405 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/263b430d84a746fcae213f8d893374b4 is 50, key is test_row_0/C:col10/1731568594602/Put/seqid=0 2024-11-14T07:16:36,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742002_1178 (size=31617) 2024-11-14T07:16:36,421 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5bf2d0523ade4619ab596355b3d8e40a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5bf2d0523ade4619ab596355b3d8e40a 2024-11-14T07:16:36,428 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into 5bf2d0523ade4619ab596355b3d8e40a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:36,428 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:36,428 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=13, startTime=1731568596288; duration=0sec 2024-11-14T07:16:36,428 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:36,428 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:36,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742003_1179 (size=12663) 2024-11-14T07:16:36,446 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/263b430d84a746fcae213f8d893374b4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/263b430d84a746fcae213f8d893374b4 2024-11-14T07:16:36,454 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into 263b430d84a746fcae213f8d893374b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:36,454 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:36,455 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=13, startTime=1731568596292; duration=0sec 2024-11-14T07:16:36,455 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:36,455 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:16:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:36,810 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142acce97db4684f32b6acf32d6a03789c_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142acce97db4684f32b6acf32d6a03789c_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5388c28a47024bb6b7bc105c2c9b7d39, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:36,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5388c28a47024bb6b7bc105c2c9b7d39 is 175, key is test_row_0/A:col10/1731568594672/Put/seqid=0 2024-11-14T07:16:36,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:36,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742004_1180 (size=31205) 2024-11-14T07:16:36,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568656935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:37,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568657052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568657262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:37,268 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5388c28a47024bb6b7bc105c2c9b7d39 2024-11-14T07:16:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/58b1e70503fd4314977aa52a04be036d is 50, key is test_row_0/B:col10/1731568594672/Put/seqid=0 2024-11-14T07:16:37,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742005_1181 (size=12251) 2024-11-14T07:16:37,333 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/58b1e70503fd4314977aa52a04be036d 2024-11-14T07:16:37,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/6590eb4be77142cdba96b0c2dc541004 is 50, key is test_row_0/C:col10/1731568594672/Put/seqid=0 2024-11-14T07:16:37,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742006_1182 (size=12251) 2024-11-14T07:16:37,376 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/6590eb4be77142cdba96b0c2dc541004 2024-11-14T07:16:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5388c28a47024bb6b7bc105c2c9b7d39 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5388c28a47024bb6b7bc105c2c9b7d39 2024-11-14T07:16:37,390 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5388c28a47024bb6b7bc105c2c9b7d39, entries=150, sequenceid=263, filesize=30.5 K 2024-11-14T07:16:37,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/58b1e70503fd4314977aa52a04be036d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/58b1e70503fd4314977aa52a04be036d 2024-11-14T07:16:37,403 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/58b1e70503fd4314977aa52a04be036d, entries=150, sequenceid=263, filesize=12.0 K 2024-11-14T07:16:37,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/6590eb4be77142cdba96b0c2dc541004 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6590eb4be77142cdba96b0c2dc541004 2024-11-14T07:16:37,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6590eb4be77142cdba96b0c2dc541004, entries=150, sequenceid=263, filesize=12.0 K 2024-11-14T07:16:37,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 26fca7a5cd511ec9993090c4fc5431d4 in 1057ms, sequenceid=263, compaction requested=false 2024-11-14T07:16:37,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:37,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:37,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T07:16:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T07:16:37,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T07:16:37,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-11-14T07:16:37,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9870 sec 2024-11-14T07:16:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:16:37,583 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:37,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T07:16:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:37,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T07:16:37,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:37,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:37,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:37,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:37,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:37,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:37,587 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:16:37,591 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:37,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:37,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114137e6958608745e7979558a434b53860_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568596917/Put/seqid=0 2024-11-14T07:16:37,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742007_1183 (size=14994) 2024-11-14T07:16:37,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:37,627 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114137e6958608745e7979558a434b53860_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114137e6958608745e7979558a434b53860_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:37,629 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/52698f5061b64e11b55113f8e0e5cd4e, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:37,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/52698f5061b64e11b55113f8e0e5cd4e is 175, key is test_row_0/A:col10/1731568596917/Put/seqid=0 2024-11-14T07:16:37,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742008_1184 (size=39949) 2024-11-14T07:16:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:16:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568657697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:37,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:37,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:37,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568657802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:37,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:37,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:16:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568658011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:38,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/52698f5061b64e11b55113f8e0e5cd4e 2024-11-14T07:16:38,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/02ef130af03c4fbeaf52debe4f3cb652 is 50, key is test_row_0/B:col10/1731568596917/Put/seqid=0 2024-11-14T07:16:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742009_1185 (size=12301) 2024-11-14T07:16:38,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:38,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:38,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:16:38,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568658322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:38,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:38,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:38,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/02ef130af03c4fbeaf52debe4f3cb652 2024-11-14T07:16:38,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/530c68b40eed4afc81cf10682f3d6147 is 50, key is test_row_0/C:col10/1731568596917/Put/seqid=0 2024-11-14T07:16:38,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742010_1186 (size=12301) 2024-11-14T07:16:38,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:16:38,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:38,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568658832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:38,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/530c68b40eed4afc81cf10682f3d6147 2024-11-14T07:16:38,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/52698f5061b64e11b55113f8e0e5cd4e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/52698f5061b64e11b55113f8e0e5cd4e 2024-11-14T07:16:38,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/52698f5061b64e11b55113f8e0e5cd4e, entries=200, sequenceid=280, filesize=39.0 K 2024-11-14T07:16:38,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/02ef130af03c4fbeaf52debe4f3cb652 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/02ef130af03c4fbeaf52debe4f3cb652 2024-11-14T07:16:38,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/02ef130af03c4fbeaf52debe4f3cb652, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T07:16:38,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/530c68b40eed4afc81cf10682f3d6147 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/530c68b40eed4afc81cf10682f3d6147 2024-11-14T07:16:38,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/530c68b40eed4afc81cf10682f3d6147, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T07:16:38,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 26fca7a5cd511ec9993090c4fc5431d4 in 1293ms, sequenceid=280, compaction requested=true 2024-11-14T07:16:38,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:38,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:38,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:38,880 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:38,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:38,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:38,880 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:38,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:38,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:38,881 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37215 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:38,881 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:38,881 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:38,881 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:38,881 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,881 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,881 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5bf2d0523ade4619ab596355b3d8e40a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5388c28a47024bb6b7bc105c2c9b7d39, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/52698f5061b64e11b55113f8e0e5cd4e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=100.4 K 2024-11-14T07:16:38,881 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c4fa755565f449d7bf00513882e91064, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/58b1e70503fd4314977aa52a04be036d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/02ef130af03c4fbeaf52debe4f3cb652] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=36.3 K 2024-11-14T07:16:38,881 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,881 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5bf2d0523ade4619ab596355b3d8e40a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5388c28a47024bb6b7bc105c2c9b7d39, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/52698f5061b64e11b55113f8e0e5cd4e] 2024-11-14T07:16:38,882 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c4fa755565f449d7bf00513882e91064, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731568594266 2024-11-14T07:16:38,882 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf2d0523ade4619ab596355b3d8e40a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731568594266 2024-11-14T07:16:38,882 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 58b1e70503fd4314977aa52a04be036d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731568594652 2024-11-14T07:16:38,882 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5388c28a47024bb6b7bc105c2c9b7d39, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731568594652 2024-11-14T07:16:38,882 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ef130af03c4fbeaf52debe4f3cb652, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568596910 2024-11-14T07:16:38,882 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52698f5061b64e11b55113f8e0e5cd4e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568596900 2024-11-14T07:16:38,890 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:38,890 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/081132b961f8457eafe21a55d356fabf is 50, key is test_row_0/B:col10/1731568596917/Put/seqid=0 2024-11-14T07:16:38,897 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:38,899 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111424173918b10d4e438b18011c1f2ba202_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:38,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742011_1187 (size=12915) 2024-11-14T07:16:38,901 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111424173918b10d4e438b18011c1f2ba202_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:38,901 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111424173918b10d4e438b18011c1f2ba202_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:38,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742012_1188 (size=4469) 2024-11-14T07:16:38,910 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#154 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:38,911 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/8bf777d098c640dfbb378fcb175b2d4a is 175, key is test_row_0/A:col10/1731568596917/Put/seqid=0 2024-11-14T07:16:38,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742013_1189 (size=31869) 2024-11-14T07:16:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:38,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:16:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:38,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114590cca067294472bbf07044eabece4b1_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568597695/Put/seqid=0 2024-11-14T07:16:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742014_1190 (size=14994) 2024-11-14T07:16:38,929 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:38,933 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114590cca067294472bbf07044eabece4b1_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114590cca067294472bbf07044eabece4b1_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:38,934 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9b79dfb96ec64db98f546d92e9d41dc0, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:38,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9b79dfb96ec64db98f546d92e9d41dc0 is 175, key is test_row_0/A:col10/1731568597695/Put/seqid=0 2024-11-14T07:16:38,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742015_1191 (size=39949) 2024-11-14T07:16:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568658942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568658945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:38,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:38,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568659052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:39,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568659052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:39,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:39,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568659262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:39,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568659272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:39,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:39,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:39,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:39,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,306 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/081132b961f8457eafe21a55d356fabf as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/081132b961f8457eafe21a55d356fabf 2024-11-14T07:16:39,312 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into 081132b961f8457eafe21a55d356fabf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:39,312 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:39,312 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=13, startTime=1731568598880; duration=0sec 2024-11-14T07:16:39,312 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:39,312 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:39,313 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:39,314 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37215 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:39,314 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:39,314 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,314 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/263b430d84a746fcae213f8d893374b4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6590eb4be77142cdba96b0c2dc541004, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/530c68b40eed4afc81cf10682f3d6147] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=36.3 K 2024-11-14T07:16:39,315 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 263b430d84a746fcae213f8d893374b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731568594266 2024-11-14T07:16:39,315 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6590eb4be77142cdba96b0c2dc541004, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731568594652 2024-11-14T07:16:39,316 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 530c68b40eed4afc81cf10682f3d6147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568596910 2024-11-14T07:16:39,320 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/8bf777d098c640dfbb378fcb175b2d4a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/8bf777d098c640dfbb378fcb175b2d4a 2024-11-14T07:16:39,326 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:39,327 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/350340ac8b604b399686e872cfb150ae is 50, key is test_row_0/C:col10/1731568596917/Put/seqid=0 2024-11-14T07:16:39,329 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into 8bf777d098c640dfbb378fcb175b2d4a(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:39,329 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:39,329 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=13, startTime=1731568598880; duration=0sec 2024-11-14T07:16:39,330 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:39,330 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:39,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742016_1192 (size=12915) 2024-11-14T07:16:39,339 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9b79dfb96ec64db98f546d92e9d41dc0 2024-11-14T07:16:39,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/88ba138345e346c8b7a6a32d6246be7c is 50, key is test_row_0/B:col10/1731568597695/Put/seqid=0 2024-11-14T07:16:39,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742017_1193 (size=12301) 2024-11-14T07:16:39,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568659572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:39,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:39,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568659583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:16:39,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,738 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/350340ac8b604b399686e872cfb150ae as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/350340ac8b604b399686e872cfb150ae 2024-11-14T07:16:39,743 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into 350340ac8b604b399686e872cfb150ae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:39,743 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:39,743 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=13, startTime=1731568598880; duration=0sec 2024-11-14T07:16:39,743 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:39,743 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:39,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/88ba138345e346c8b7a6a32d6246be7c 2024-11-14T07:16:39,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/e5cb4bdfeb8d440cbfc37842c3f5db5b is 50, key is test_row_0/C:col10/1731568597695/Put/seqid=0 2024-11-14T07:16:39,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742018_1194 (size=12301) 2024-11-14T07:16:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568659852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:39,856 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:39,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:39,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:40,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:40,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:40,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:40,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568660082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:40,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568660092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:40,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:40,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:40,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/e5cb4bdfeb8d440cbfc37842c3f5db5b 2024-11-14T07:16:40,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9b79dfb96ec64db98f546d92e9d41dc0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9b79dfb96ec64db98f546d92e9d41dc0 2024-11-14T07:16:40,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9b79dfb96ec64db98f546d92e9d41dc0, entries=200, sequenceid=299, filesize=39.0 K 2024-11-14T07:16:40,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/88ba138345e346c8b7a6a32d6246be7c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/88ba138345e346c8b7a6a32d6246be7c 2024-11-14T07:16:40,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/88ba138345e346c8b7a6a32d6246be7c, entries=150, sequenceid=299, filesize=12.0 K 2024-11-14T07:16:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/e5cb4bdfeb8d440cbfc37842c3f5db5b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/e5cb4bdfeb8d440cbfc37842c3f5db5b 2024-11-14T07:16:40,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/e5cb4bdfeb8d440cbfc37842c3f5db5b, entries=150, sequenceid=299, filesize=12.0 K 2024-11-14T07:16:40,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 26fca7a5cd511ec9993090c4fc5431d4 in 1316ms, sequenceid=299, compaction requested=false 2024-11-14T07:16:40,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:40,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:16:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:40,348 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:16:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:40,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aae1bd8a7cbb4ac589a9b6b493d31bf3_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568598939/Put/seqid=0 2024-11-14T07:16:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742019_1195 (size=12454) 2024-11-14T07:16:40,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:40,771 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aae1bd8a7cbb4ac589a9b6b493d31bf3_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aae1bd8a7cbb4ac589a9b6b493d31bf3_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:40,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b675391eb3824edd9f415231d5ab7d4e, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:40,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b675391eb3824edd9f415231d5ab7d4e is 175, key is test_row_0/A:col10/1731568598939/Put/seqid=0 2024-11-14T07:16:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742020_1196 (size=31255) 2024-11-14T07:16:40,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b675391eb3824edd9f415231d5ab7d4e 2024-11-14T07:16:40,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/a99c6933caaa4b44b2fbc7c0bc27680f is 50, key is test_row_0/B:col10/1731568598939/Put/seqid=0 2024-11-14T07:16:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742021_1197 (size=12301) 2024-11-14T07:16:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:41,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:41,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568661138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568661139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/a99c6933caaa4b44b2fbc7c0bc27680f 2024-11-14T07:16:41,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568661241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568661252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/6a2a7859f09f43b1b8f0b7f51481b199 is 50, key is test_row_0/C:col10/1731568598939/Put/seqid=0 2024-11-14T07:16:41,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742022_1198 (size=12301) 2024-11-14T07:16:41,273 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/6a2a7859f09f43b1b8f0b7f51481b199 2024-11-14T07:16:41,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/b675391eb3824edd9f415231d5ab7d4e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b675391eb3824edd9f415231d5ab7d4e 2024-11-14T07:16:41,289 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b675391eb3824edd9f415231d5ab7d4e, entries=150, sequenceid=315, filesize=30.5 K 2024-11-14T07:16:41,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/a99c6933caaa4b44b2fbc7c0bc27680f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/a99c6933caaa4b44b2fbc7c0bc27680f 2024-11-14T07:16:41,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/a99c6933caaa4b44b2fbc7c0bc27680f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:16:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/6a2a7859f09f43b1b8f0b7f51481b199 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6a2a7859f09f43b1b8f0b7f51481b199 2024-11-14T07:16:41,306 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6a2a7859f09f43b1b8f0b7f51481b199, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:16:41,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 26fca7a5cd511ec9993090c4fc5431d4 in 960ms, sequenceid=315, compaction requested=true 2024-11-14T07:16:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T07:16:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T07:16:41,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T07:16:41,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7190 sec 2024-11-14T07:16:41,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.7290 sec 2024-11-14T07:16:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:41,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:16:41,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:41,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:41,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:41,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:41,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:41,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:41,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111463c88de3c0434ad3946c3c86b39848f2_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568601137/Put/seqid=0 2024-11-14T07:16:41,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742023_1199 (size=14994) 2024-11-14T07:16:41,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568661477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568661478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568661592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568661592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:16:41,743 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:41,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T07:16:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:16:41,747 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:41,748 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:41,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568661801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568661801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:16:41,866 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:41,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111463c88de3c0434ad3946c3c86b39848f2_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111463c88de3c0434ad3946c3c86b39848f2_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:41,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ec66e1e8eac34785b123f2e2c8426a15, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:41,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ec66e1e8eac34785b123f2e2c8426a15 is 175, key is test_row_0/A:col10/1731568601137/Put/seqid=0 2024-11-14T07:16:41,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46216 deadline: 1731568661881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:41,885 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 26fca7a5cd511ec9993090c4fc5431d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:16:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:41,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T07:16:41,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:41,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:41,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:41,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:41,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:41,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742024_1200 (size=39949) 2024-11-14T07:16:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:41,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ec66e1e8eac34785b123f2e2c8426a15 2024-11-14T07:16:41,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/48ad15a121b94f1aa21611bc68414c25 is 50, key is test_row_0/B:col10/1731568601137/Put/seqid=0 2024-11-14T07:16:41,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742025_1201 (size=12301) 2024-11-14T07:16:42,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T07:16:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:16:42,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568662112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568662112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:42,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T07:16:42,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/48ad15a121b94f1aa21611bc68414c25 2024-11-14T07:16:42,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/5415320a2d2f4372b35f56e108cf7d50 is 50, key is test_row_0/C:col10/1731568601137/Put/seqid=0 2024-11-14T07:16:42,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742026_1202 (size=12301) 2024-11-14T07:16:42,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/5415320a2d2f4372b35f56e108cf7d50 2024-11-14T07:16:42,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/ec66e1e8eac34785b123f2e2c8426a15 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ec66e1e8eac34785b123f2e2c8426a15 2024-11-14T07:16:42,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T07:16:42,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:42,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:42,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ec66e1e8eac34785b123f2e2c8426a15, entries=200, sequenceid=335, filesize=39.0 K 2024-11-14T07:16:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:16:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/48ad15a121b94f1aa21611bc68414c25 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/48ad15a121b94f1aa21611bc68414c25 2024-11-14T07:16:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/48ad15a121b94f1aa21611bc68414c25, entries=150, sequenceid=335, filesize=12.0 K 2024-11-14T07:16:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/5415320a2d2f4372b35f56e108cf7d50 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5415320a2d2f4372b35f56e108cf7d50 2024-11-14T07:16:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5415320a2d2f4372b35f56e108cf7d50, entries=150, sequenceid=335, filesize=12.0 K 2024-11-14T07:16:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 26fca7a5cd511ec9993090c4fc5431d4 in 936ms, sequenceid=335, compaction requested=true 2024-11-14T07:16:42,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:42,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:42,391 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:42,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:42,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:42,391 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:42,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:42,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,393 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:42,393 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:42,393 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:42,393 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:42,393 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,393 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,393 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/081132b961f8457eafe21a55d356fabf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/88ba138345e346c8b7a6a32d6246be7c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/a99c6933caaa4b44b2fbc7c0bc27680f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/48ad15a121b94f1aa21611bc68414c25] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=48.7 K 2024-11-14T07:16:42,393 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/8bf777d098c640dfbb378fcb175b2d4a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9b79dfb96ec64db98f546d92e9d41dc0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b675391eb3824edd9f415231d5ab7d4e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ec66e1e8eac34785b123f2e2c8426a15] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=139.7 K 2024-11-14T07:16:42,393 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,393 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/8bf777d098c640dfbb378fcb175b2d4a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9b79dfb96ec64db98f546d92e9d41dc0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b675391eb3824edd9f415231d5ab7d4e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ec66e1e8eac34785b123f2e2c8426a15] 2024-11-14T07:16:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,393 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 081132b961f8457eafe21a55d356fabf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568596910 2024-11-14T07:16:42,394 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bf777d098c640dfbb378fcb175b2d4a, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568596910 2024-11-14T07:16:42,394 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ba138345e346c8b7a6a32d6246be7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731568597667 2024-11-14T07:16:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,394 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b79dfb96ec64db98f546d92e9d41dc0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731568597667 2024-11-14T07:16:42,394 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a99c6933caaa4b44b2fbc7c0bc27680f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731568598922 2024-11-14T07:16:42,394 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting b675391eb3824edd9f415231d5ab7d4e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731568598922 2024-11-14T07:16:42,394 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 48ad15a121b94f1aa21611bc68414c25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731568601131 2024-11-14T07:16:42,394 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec66e1e8eac34785b123f2e2c8426a15, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731568601116 2024-11-14T07:16:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,403 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,405 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#165 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,406 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/1ec31250d202409eb2821303251ed056 is 50, key is test_row_0/B:col10/1731568601137/Put/seqid=0 2024-11-14T07:16:42,406 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a16ca7687b1d4737a3912b642efec202_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,409 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a16ca7687b1d4737a3912b642efec202_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,409 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a16ca7687b1d4737a3912b642efec202_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742027_1203 (size=13051) 2024-11-14T07:16:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742028_1204 (size=4469) 2024-11-14T07:16:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T07:16:42,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,515 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:16:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111425e57965c7f94c498b51557dd4fdc61c_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568601477/Put/seqid=0 2024-11-14T07:16:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742029_1205 (size=9914) 2024-11-14T07:16:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,551 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111425e57965c7f94c498b51557dd4fdc61c_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111425e57965c7f94c498b51557dd4fdc61c_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a0117efad54b43dfb202be226fe6a248, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a0117efad54b43dfb202be226fe6a248 is 175, key is test_row_0/A:col10/1731568601477/Put/seqid=0 2024-11-14T07:16:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742030_1206 (size=22561) 2024-11-14T07:16:42,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a0117efad54b43dfb202be226fe6a248 2024-11-14T07:16:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/2e907aee9c2242ce95e2d528ef0c5c05 is 50, key is test_row_0/B:col10/1731568601477/Put/seqid=0 2024-11-14T07:16:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742031_1207 (size=9857) 2024-11-14T07:16:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568662672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:42,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568662674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:42,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568662782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568662782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:42,825 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/1ec31250d202409eb2821303251ed056 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/1ec31250d202409eb2821303251ed056 2024-11-14T07:16:42,830 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into 1ec31250d202409eb2821303251ed056(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:42,830 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:42,830 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=12, startTime=1731568602391; duration=0sec 2024-11-14T07:16:42,830 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:42,830 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:42,830 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:16:42,831 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:16:42,832 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:42,832 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:42,832 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/350340ac8b604b399686e872cfb150ae, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/e5cb4bdfeb8d440cbfc37842c3f5db5b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6a2a7859f09f43b1b8f0b7f51481b199, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5415320a2d2f4372b35f56e108cf7d50] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=48.7 K 2024-11-14T07:16:42,832 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 350340ac8b604b399686e872cfb150ae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568596910 2024-11-14T07:16:42,832 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e5cb4bdfeb8d440cbfc37842c3f5db5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731568597667 2024-11-14T07:16:42,833 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2a7859f09f43b1b8f0b7f51481b199, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731568598922 2024-11-14T07:16:42,833 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5415320a2d2f4372b35f56e108cf7d50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731568601131 2024-11-14T07:16:42,839 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:42,840 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9d648816dc704ff6a39496dfe532775d is 175, key is test_row_0/A:col10/1731568601137/Put/seqid=0 2024-11-14T07:16:42,841 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:42,842 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/ffae1641892841bfb790b12a632454ba is 50, key is test_row_0/C:col10/1731568601137/Put/seqid=0 2024-11-14T07:16:42,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742032_1208 (size=32005) 2024-11-14T07:16:42,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742033_1209 (size=13051) 2024-11-14T07:16:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:16:42,980 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/2e907aee9c2242ce95e2d528ef0c5c05 2024-11-14T07:16:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568662992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568662992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:43,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/99c849c65f71492498c827c70e47eb59 is 50, key is test_row_0/C:col10/1731568601477/Put/seqid=0 2024-11-14T07:16:43,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742034_1210 (size=9857) 2024-11-14T07:16:43,249 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/9d648816dc704ff6a39496dfe532775d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9d648816dc704ff6a39496dfe532775d 2024-11-14T07:16:43,251 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/ffae1641892841bfb790b12a632454ba as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ffae1641892841bfb790b12a632454ba 2024-11-14T07:16:43,255 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into 9d648816dc704ff6a39496dfe532775d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:43,255 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:43,255 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=12, startTime=1731568602391; duration=0sec 2024-11-14T07:16:43,255 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:43,255 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:43,256 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into ffae1641892841bfb790b12a632454ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:43,256 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:43,256 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=12, startTime=1731568602391; duration=0sec 2024-11-14T07:16:43,256 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:43,256 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:43,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568663301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:43,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568663301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:43,451 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/99c849c65f71492498c827c70e47eb59 2024-11-14T07:16:43,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/a0117efad54b43dfb202be226fe6a248 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0117efad54b43dfb202be226fe6a248 2024-11-14T07:16:43,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0117efad54b43dfb202be226fe6a248, entries=100, sequenceid=347, filesize=22.0 K 2024-11-14T07:16:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/2e907aee9c2242ce95e2d528ef0c5c05 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/2e907aee9c2242ce95e2d528ef0c5c05 2024-11-14T07:16:43,468 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/2e907aee9c2242ce95e2d528ef0c5c05, entries=100, sequenceid=347, filesize=9.6 K 2024-11-14T07:16:43,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/99c849c65f71492498c827c70e47eb59 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/99c849c65f71492498c827c70e47eb59 2024-11-14T07:16:43,477 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/99c849c65f71492498c827c70e47eb59, entries=100, sequenceid=347, filesize=9.6 K 2024-11-14T07:16:43,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 26fca7a5cd511ec9993090c4fc5431d4 in 963ms, sequenceid=347, compaction requested=false 2024-11-14T07:16:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T07:16:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T07:16:43,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T07:16:43,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7320 sec 2024-11-14T07:16:43,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7390 sec 2024-11-14T07:16:43,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568602657, max timestamp=1731568603812Num uniques 859; 2024-11-14T07:16:43,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:16:43,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568602657, max timestamp=1731568603812Num uniques 859; 2024-11-14T07:16:43,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568602657, max timestamp=1731568603812Num uniques 859; 2024-11-14T07:16:43,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:16:43,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:16:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:43,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-14T07:16:43,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:43,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:43,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:43,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:43,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111485cd016060a54c589e2c240271dd39fb_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568603822/Put/seqid=0 2024-11-14T07:16:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742035_1211 (size=17534) 2024-11-14T07:16:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:16:43,892 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:43,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-14T07:16:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:16:43,893 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:43,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:43,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:43,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568603878, max timestamp=1731568603915Num uniques 859; 2024-11-14T07:16:43,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:43,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568603878, max timestamp=1731568603915Num uniques 859; 2024-11-14T07:16:43,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568603878, max timestamp=1731568603915Num uniques 859; 2024-11-14T07:16:43,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:43,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:16:43,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568663956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:43,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:43,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568663957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:16:44,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568664061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568664061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:44,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:16:44,261 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:44,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111485cd016060a54c589e2c240271dd39fb_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111485cd016060a54c589e2c240271dd39fb_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:44,266 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/182642059f2c479fa341dc82b898f128, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:44,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/182642059f2c479fa341dc82b898f128 is 175, key is test_row_0/A:col10/1731568603822/Put/seqid=0 2024-11-14T07:16:44,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742036_1212 (size=48639) 2024-11-14T07:16:44,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568664271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568664271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:44,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:44,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:44,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:44,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:16:44,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568664582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:44,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568664582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:44,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:44,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:44,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/182642059f2c479fa341dc82b898f128 2024-11-14T07:16:44,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/51e0f6bc1a1040468a26267681db2d79 is 50, key is test_row_0/B:col10/1731568603822/Put/seqid=0 2024-11-14T07:16:44,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742037_1213 (size=12301) 2024-11-14T07:16:44,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:44,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:44,991 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-14T07:16:44,991 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:44,991 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:44,991 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:44,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:44,993 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-14T07:16:44,993 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:44,993 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:44,993 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:44,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:44,994 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-14T07:16:44,994 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:44,994 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:44,994 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:44,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:45,001 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-14T07:16:45,001 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:45,001 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,001 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:16:45,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/51e0f6bc1a1040468a26267681db2d79 2024-11-14T07:16:45,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:45,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46280 deadline: 1731568665092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1731568665092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:16:45,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/de07229648564393984c027b81b3b8f4 is 50, key is test_row_0/C:col10/1731568603822/Put/seqid=0 2024-11-14T07:16:45,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742038_1214 (size=12301) 2024-11-14T07:16:45,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:45,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:45,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:45,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/de07229648564393984c027b81b3b8f4 2024-11-14T07:16:45,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/182642059f2c479fa341dc82b898f128 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/182642059f2c479fa341dc82b898f128 2024-11-14T07:16:45,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/182642059f2c479fa341dc82b898f128, entries=250, sequenceid=375, filesize=47.5 K 2024-11-14T07:16:45,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/51e0f6bc1a1040468a26267681db2d79 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/51e0f6bc1a1040468a26267681db2d79 2024-11-14T07:16:45,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/51e0f6bc1a1040468a26267681db2d79, entries=150, sequenceid=375, filesize=12.0 K 2024-11-14T07:16:45,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/de07229648564393984c027b81b3b8f4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/de07229648564393984c027b81b3b8f4 2024-11-14T07:16:45,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/de07229648564393984c027b81b3b8f4, entries=150, sequenceid=375, filesize=12.0 K 2024-11-14T07:16:45,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 26fca7a5cd511ec9993090c4fc5431d4 in 1695ms, sequenceid=375, compaction requested=true 2024-11-14T07:16:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:45,521 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26fca7a5cd511ec9993090c4fc5431d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:45,521 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:45,522 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:45,522 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:45,522 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/B is initiating minor compaction (all files) 2024-11-14T07:16:45,522 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/A is initiating minor compaction (all files) 2024-11-14T07:16:45,522 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/B in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,522 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/A in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,522 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/1ec31250d202409eb2821303251ed056, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/2e907aee9c2242ce95e2d528ef0c5c05, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/51e0f6bc1a1040468a26267681db2d79] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=34.4 K 2024-11-14T07:16:45,522 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9d648816dc704ff6a39496dfe532775d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0117efad54b43dfb202be226fe6a248, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/182642059f2c479fa341dc82b898f128] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=100.8 K 2024-11-14T07:16:45,523 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,523 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9d648816dc704ff6a39496dfe532775d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0117efad54b43dfb202be226fe6a248, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/182642059f2c479fa341dc82b898f128] 2024-11-14T07:16:45,523 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec31250d202409eb2821303251ed056, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731568601131 2024-11-14T07:16:45,523 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d648816dc704ff6a39496dfe532775d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731568601131 2024-11-14T07:16:45,523 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e907aee9c2242ce95e2d528ef0c5c05, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731568601469 2024-11-14T07:16:45,523 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0117efad54b43dfb202be226fe6a248, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731568601469 2024-11-14T07:16:45,524 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e0f6bc1a1040468a26267681db2d79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731568603814 2024-11-14T07:16:45,524 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 182642059f2c479fa341dc82b898f128, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731568603812 2024-11-14T07:16:45,531 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:45,532 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/b78f9e17dc4e4bbbb78782736170580d is 50, key is test_row_0/B:col10/1731568603822/Put/seqid=0 2024-11-14T07:16:45,533 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:45,535 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111460a219eff3b044f1b2dfb7704573fe39_26fca7a5cd511ec9993090c4fc5431d4 store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742039_1215 (size=13153) 2024-11-14T07:16:45,554 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111460a219eff3b044f1b2dfb7704573fe39_26fca7a5cd511ec9993090c4fc5431d4, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:45,554 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111460a219eff3b044f1b2dfb7704573fe39_26fca7a5cd511ec9993090c4fc5431d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:45,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742040_1216 (size=4469) 2024-11-14T07:16:45,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:16:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T07:16:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:45,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:45,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:45,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:45,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:45,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114965c02c763c6424aa31564973f07ace5_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568603955/Put/seqid=0 2024-11-14T07:16:45,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742041_1217 (size=12454) 2024-11-14T07:16:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:45,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. as already flushing 2024-11-14T07:16:45,733 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-14T07:16:45,733 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:45,733 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,734 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:45,773 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-14T07:16:45,773 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:45,773 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,773 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:45,893 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-14T07:16:45,893 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:45,893 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,893 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:45,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:45,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/b78f9e17dc4e4bbbb78782736170580d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b78f9e17dc4e4bbbb78782736170580d 2024-11-14T07:16:45,950 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/B of 26fca7a5cd511ec9993090c4fc5431d4 into b78f9e17dc4e4bbbb78782736170580d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:45,950 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:45,950 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/B, priority=13, startTime=1731568605521; duration=0sec 2024-11-14T07:16:45,950 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:45,950 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:B 2024-11-14T07:16:45,950 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:45,951 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:45,951 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 26fca7a5cd511ec9993090c4fc5431d4/C is initiating minor compaction (all files) 2024-11-14T07:16:45,951 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26fca7a5cd511ec9993090c4fc5431d4/C in TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:45,952 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ffae1641892841bfb790b12a632454ba, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/99c849c65f71492498c827c70e47eb59, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/de07229648564393984c027b81b3b8f4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp, totalSize=34.4 K 2024-11-14T07:16:45,952 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ffae1641892841bfb790b12a632454ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731568601131 2024-11-14T07:16:45,952 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c849c65f71492498c827c70e47eb59, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731568601469 2024-11-14T07:16:45,953 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting de07229648564393984c027b81b3b8f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731568603814 2024-11-14T07:16:45,958 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#A#compaction#175 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:45,959 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5201c4f574ae42018b8c9ef0dd2ce5c6 is 175, key is test_row_0/A:col10/1731568603822/Put/seqid=0 2024-11-14T07:16:45,960 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26fca7a5cd511ec9993090c4fc5431d4#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:45,960 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/3da31818e16d4cc38074b4102b4047bc is 50, key is test_row_0/C:col10/1731568603822/Put/seqid=0 2024-11-14T07:16:45,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742042_1218 (size=32107) 2024-11-14T07:16:45,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742043_1219 (size=13153) 2024-11-14T07:16:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:45,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114965c02c763c6424aa31564973f07ace5_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114965c02c763c6424aa31564973f07ace5_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:45,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/f2e0cf1a1e1f43e299a47bdb80cf7845, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:45,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/f2e0cf1a1e1f43e299a47bdb80cf7845 is 175, key is test_row_0/A:col10/1731568603955/Put/seqid=0 2024-11-14T07:16:46,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742044_1220 (size=31255) 2024-11-14T07:16:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:16:46,103 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-14T07:16:46,103 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:46,103 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:46,104 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:46,104 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-14T07:16:46,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:46,104 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:16:46,104 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:46,104 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:46,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:46,375 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/5201c4f574ae42018b8c9ef0dd2ce5c6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5201c4f574ae42018b8c9ef0dd2ce5c6 2024-11-14T07:16:46,376 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/3da31818e16d4cc38074b4102b4047bc as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/3da31818e16d4cc38074b4102b4047bc 2024-11-14T07:16:46,380 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/A of 26fca7a5cd511ec9993090c4fc5431d4 into 5201c4f574ae42018b8c9ef0dd2ce5c6(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:46,380 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26fca7a5cd511ec9993090c4fc5431d4/C of 26fca7a5cd511ec9993090c4fc5431d4 into 3da31818e16d4cc38074b4102b4047bc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:46,380 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:46,380 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:46,381 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/A, priority=13, startTime=1731568605521; duration=0sec 2024-11-14T07:16:46,381 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., storeName=26fca7a5cd511ec9993090c4fc5431d4/C, priority=13, startTime=1731568605521; duration=0sec 2024-11-14T07:16:46,381 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:46,381 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:A 2024-11-14T07:16:46,381 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:46,381 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26fca7a5cd511ec9993090c4fc5431d4:C 2024-11-14T07:16:46,403 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/f2e0cf1a1e1f43e299a47bdb80cf7845 2024-11-14T07:16:46,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/409b6b419e7843a8bfdb9b5e82a1fda9 is 50, key is test_row_0/B:col10/1731568603955/Put/seqid=0 2024-11-14T07:16:46,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742045_1221 (size=12301) 2024-11-14T07:16:46,815 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/409b6b419e7843a8bfdb9b5e82a1fda9 2024-11-14T07:16:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/f932ae93a687493b8113c12fc106f82c is 50, key is test_row_0/C:col10/1731568603955/Put/seqid=0 2024-11-14T07:16:46,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742046_1222 (size=12301) 2024-11-14T07:16:47,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/f932ae93a687493b8113c12fc106f82c 2024-11-14T07:16:47,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/f2e0cf1a1e1f43e299a47bdb80cf7845 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/f2e0cf1a1e1f43e299a47bdb80cf7845 2024-11-14T07:16:47,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/f2e0cf1a1e1f43e299a47bdb80cf7845, entries=150, sequenceid=410, filesize=30.5 K 2024-11-14T07:16:47,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/409b6b419e7843a8bfdb9b5e82a1fda9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/409b6b419e7843a8bfdb9b5e82a1fda9 2024-11-14T07:16:47,257 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/409b6b419e7843a8bfdb9b5e82a1fda9, entries=150, sequenceid=410, filesize=12.0 K 2024-11-14T07:16:47,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/f932ae93a687493b8113c12fc106f82c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f932ae93a687493b8113c12fc106f82c 2024-11-14T07:16:47,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f932ae93a687493b8113c12fc106f82c, entries=150, sequenceid=410, filesize=12.0 K 2024-11-14T07:16:47,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=33.54 KB/34350 for 26fca7a5cd511ec9993090c4fc5431d4 in 1685ms, sequenceid=410, compaction requested=false 2024-11-14T07:16:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 26fca7a5cd511ec9993090c4fc5431d4: 2024-11-14T07:16:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-14T07:16:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-14T07:16:47,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T07:16:47,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3700 sec 2024-11-14T07:16:47,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.3730 sec 2024-11-14T07:16:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:16:48,063 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2582 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2558 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1136 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3408 rows 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1141 2024-11-14T07:16:48,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3422 rows 2024-11-14T07:16:48,063 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:16:48,064 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:16:48,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:48,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:48,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:16:48,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:16:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:16:48,072 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568608072"}]},"ts":"1731568608072"} 2024-11-14T07:16:48,074 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:16:48,074 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:16:48,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:16:48,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, UNASSIGN}] 2024-11-14T07:16:48,077 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, UNASSIGN 2024-11-14T07:16:48,078 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:48,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, UNASSIGN because future has completed 2024-11-14T07:16:48,080 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:16:48,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:16:48,232 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 26fca7a5cd511ec9993090c4fc5431d4, disabling compactions & flushes 2024-11-14T07:16:48,233 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. after waiting 0 ms 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:48,233 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 26fca7a5cd511ec9993090c4fc5431d4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=A 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=B 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26fca7a5cd511ec9993090c4fc5431d4, store=C 2024-11-14T07:16:48,233 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:48,239 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b782d613a3c74130b661f55dedd93354_26fca7a5cd511ec9993090c4fc5431d4 is 50, key is test_row_0/A:col10/1731568606102/Put/seqid=0 2024-11-14T07:16:48,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742047_1223 (size=12454) 2024-11-14T07:16:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:16:48,497 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:16:48,645 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:48,653 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b782d613a3c74130b661f55dedd93354_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b782d613a3c74130b661f55dedd93354_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:48,655 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/515ee20ded44407fa6b873385a07d03c, store: [table=TestAcidGuarantees family=A region=26fca7a5cd511ec9993090c4fc5431d4] 2024-11-14T07:16:48,656 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/515ee20ded44407fa6b873385a07d03c is 175, key is test_row_0/A:col10/1731568606102/Put/seqid=0 2024-11-14T07:16:48,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742048_1224 (size=31255) 2024-11-14T07:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:16:49,061 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/515ee20ded44407fa6b873385a07d03c 2024-11-14T07:16:49,069 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/39c4d62235214ed1ab61da9ccffc3fb5 is 50, key is test_row_0/B:col10/1731568606102/Put/seqid=0 2024-11-14T07:16:49,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742049_1225 (size=12301) 2024-11-14T07:16:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:16:49,473 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/39c4d62235214ed1ab61da9ccffc3fb5 2024-11-14T07:16:49,485 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/ef6cde27edec43e298b50a9fe4012b1c is 50, key is test_row_0/C:col10/1731568606102/Put/seqid=0 2024-11-14T07:16:49,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742050_1226 (size=12301) 2024-11-14T07:16:49,890 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/ef6cde27edec43e298b50a9fe4012b1c 2024-11-14T07:16:49,899 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/A/515ee20ded44407fa6b873385a07d03c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/515ee20ded44407fa6b873385a07d03c 2024-11-14T07:16:49,904 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/515ee20ded44407fa6b873385a07d03c, entries=150, sequenceid=421, filesize=30.5 K 2024-11-14T07:16:49,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/B/39c4d62235214ed1ab61da9ccffc3fb5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/39c4d62235214ed1ab61da9ccffc3fb5 2024-11-14T07:16:49,907 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/39c4d62235214ed1ab61da9ccffc3fb5, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T07:16:49,908 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/.tmp/C/ef6cde27edec43e298b50a9fe4012b1c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ef6cde27edec43e298b50a9fe4012b1c 2024-11-14T07:16:49,911 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ef6cde27edec43e298b50a9fe4012b1c, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T07:16:49,912 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 26fca7a5cd511ec9993090c4fc5431d4 in 1679ms, sequenceid=421, compaction requested=true 2024-11-14T07:16:49,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9af5d02133484c4d9060bc599756f466, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/6bd1e92e64724c929fe8d0c8bc945a33, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/12270f52b16f458b991c727654d1b4db, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0306a14ea0d4754bdbc23e55c7fa3d6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c481577310754948adfb103039480529, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c488d373b1054c47ac4178e5eb71431a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eec383c16b534d12865ed0a93e75aef1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eae70adf754948838ff27d57727172e0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a888cb4036bf46698d3d1facfabf8272, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/2c123ec839874d1ca994e0837e39adc2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b12c47a08a0a4775b62ab69cc09bf8fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ac97be209fdc402eb239770d2c10505b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/35148dca98e7403f9cdf8daa2fc73e65, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9feabc319d1543a7a3aed94b836dfb0e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5bf2d0523ade4619ab596355b3d8e40a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/0ddbacd254f44a6fb6e3bec425f870a2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5388c28a47024bb6b7bc105c2c9b7d39, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/52698f5061b64e11b55113f8e0e5cd4e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/8bf777d098c640dfbb378fcb175b2d4a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9b79dfb96ec64db98f546d92e9d41dc0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b675391eb3824edd9f415231d5ab7d4e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ec66e1e8eac34785b123f2e2c8426a15, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9d648816dc704ff6a39496dfe532775d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0117efad54b43dfb202be226fe6a248, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/182642059f2c479fa341dc82b898f128] to archive 2024-11-14T07:16:49,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:16:49,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9af5d02133484c4d9060bc599756f466 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9af5d02133484c4d9060bc599756f466 2024-11-14T07:16:49,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/6bd1e92e64724c929fe8d0c8bc945a33 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/6bd1e92e64724c929fe8d0c8bc945a33 2024-11-14T07:16:49,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/12270f52b16f458b991c727654d1b4db to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/12270f52b16f458b991c727654d1b4db 2024-11-14T07:16:49,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0306a14ea0d4754bdbc23e55c7fa3d6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0306a14ea0d4754bdbc23e55c7fa3d6 2024-11-14T07:16:49,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c481577310754948adfb103039480529 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c481577310754948adfb103039480529 2024-11-14T07:16:49,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c488d373b1054c47ac4178e5eb71431a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/c488d373b1054c47ac4178e5eb71431a 2024-11-14T07:16:49,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eec383c16b534d12865ed0a93e75aef1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eec383c16b534d12865ed0a93e75aef1 2024-11-14T07:16:49,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eae70adf754948838ff27d57727172e0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/eae70adf754948838ff27d57727172e0 2024-11-14T07:16:49,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a888cb4036bf46698d3d1facfabf8272 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a888cb4036bf46698d3d1facfabf8272 2024-11-14T07:16:49,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/2c123ec839874d1ca994e0837e39adc2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/2c123ec839874d1ca994e0837e39adc2 2024-11-14T07:16:49,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b12c47a08a0a4775b62ab69cc09bf8fb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b12c47a08a0a4775b62ab69cc09bf8fb 2024-11-14T07:16:49,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ac97be209fdc402eb239770d2c10505b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ac97be209fdc402eb239770d2c10505b 2024-11-14T07:16:49,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/35148dca98e7403f9cdf8daa2fc73e65 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/35148dca98e7403f9cdf8daa2fc73e65 2024-11-14T07:16:49,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9feabc319d1543a7a3aed94b836dfb0e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9feabc319d1543a7a3aed94b836dfb0e 2024-11-14T07:16:49,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5bf2d0523ade4619ab596355b3d8e40a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5bf2d0523ade4619ab596355b3d8e40a 2024-11-14T07:16:49,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/0ddbacd254f44a6fb6e3bec425f870a2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/0ddbacd254f44a6fb6e3bec425f870a2 2024-11-14T07:16:49,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5388c28a47024bb6b7bc105c2c9b7d39 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5388c28a47024bb6b7bc105c2c9b7d39 2024-11-14T07:16:49,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/52698f5061b64e11b55113f8e0e5cd4e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/52698f5061b64e11b55113f8e0e5cd4e 2024-11-14T07:16:49,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/8bf777d098c640dfbb378fcb175b2d4a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/8bf777d098c640dfbb378fcb175b2d4a 2024-11-14T07:16:49,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9b79dfb96ec64db98f546d92e9d41dc0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9b79dfb96ec64db98f546d92e9d41dc0 2024-11-14T07:16:49,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b675391eb3824edd9f415231d5ab7d4e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/b675391eb3824edd9f415231d5ab7d4e 2024-11-14T07:16:49,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ec66e1e8eac34785b123f2e2c8426a15 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/ec66e1e8eac34785b123f2e2c8426a15 2024-11-14T07:16:49,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9d648816dc704ff6a39496dfe532775d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/9d648816dc704ff6a39496dfe532775d 2024-11-14T07:16:49,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0117efad54b43dfb202be226fe6a248 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/a0117efad54b43dfb202be226fe6a248 2024-11-14T07:16:49,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/182642059f2c479fa341dc82b898f128 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/182642059f2c479fa341dc82b898f128 2024-11-14T07:16:49,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/95fea476c6c6416f8e3bb22874e71a4d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/07d0afd057574843a67713995af3d155, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8b28ddd826d64feaba9c6cbc49442d8c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c80461dc07f340efba7d78d70109cbb0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/ebe442317d5349039d3afbdbfc5af3eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/6e0db8c62c094198acf8abffe64890d5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b662bf506fcf42c9b8b45777cdd5b599, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8f3d4345e56e42e8b7d316a2378c54ad, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f44b6aeaf92a4515acf37c6556add5bb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e6aec13dafb44279baeab46407491269, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/bb49dd8c9b4d4070bfc268258300765f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/0a3d3298a84e4e6c89350b1e1d655934, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e9c24141e81d40fcb52e4dde4a64e07c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/fd131b6b17db446f83665b367f3a7ae1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c4fa755565f449d7bf00513882e91064, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f70461ce012b40c9bbd72f4449ad371c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/58b1e70503fd4314977aa52a04be036d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/081132b961f8457eafe21a55d356fabf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/02ef130af03c4fbeaf52debe4f3cb652, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/88ba138345e346c8b7a6a32d6246be7c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/a99c6933caaa4b44b2fbc7c0bc27680f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/1ec31250d202409eb2821303251ed056, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/48ad15a121b94f1aa21611bc68414c25, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/2e907aee9c2242ce95e2d528ef0c5c05, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/51e0f6bc1a1040468a26267681db2d79] to archive 2024-11-14T07:16:49,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:16:49,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/95fea476c6c6416f8e3bb22874e71a4d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/95fea476c6c6416f8e3bb22874e71a4d 2024-11-14T07:16:49,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/07d0afd057574843a67713995af3d155 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/07d0afd057574843a67713995af3d155 2024-11-14T07:16:49,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8b28ddd826d64feaba9c6cbc49442d8c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8b28ddd826d64feaba9c6cbc49442d8c 2024-11-14T07:16:49,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c80461dc07f340efba7d78d70109cbb0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c80461dc07f340efba7d78d70109cbb0 2024-11-14T07:16:49,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/ebe442317d5349039d3afbdbfc5af3eb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/ebe442317d5349039d3afbdbfc5af3eb 2024-11-14T07:16:49,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/6e0db8c62c094198acf8abffe64890d5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/6e0db8c62c094198acf8abffe64890d5 2024-11-14T07:16:49,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b662bf506fcf42c9b8b45777cdd5b599 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b662bf506fcf42c9b8b45777cdd5b599 2024-11-14T07:16:49,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8f3d4345e56e42e8b7d316a2378c54ad to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/8f3d4345e56e42e8b7d316a2378c54ad 2024-11-14T07:16:49,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f44b6aeaf92a4515acf37c6556add5bb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f44b6aeaf92a4515acf37c6556add5bb 2024-11-14T07:16:49,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e6aec13dafb44279baeab46407491269 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e6aec13dafb44279baeab46407491269 2024-11-14T07:16:49,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/bb49dd8c9b4d4070bfc268258300765f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/bb49dd8c9b4d4070bfc268258300765f 2024-11-14T07:16:49,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/0a3d3298a84e4e6c89350b1e1d655934 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/0a3d3298a84e4e6c89350b1e1d655934 2024-11-14T07:16:49,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e9c24141e81d40fcb52e4dde4a64e07c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/e9c24141e81d40fcb52e4dde4a64e07c 2024-11-14T07:16:49,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/fd131b6b17db446f83665b367f3a7ae1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/fd131b6b17db446f83665b367f3a7ae1 2024-11-14T07:16:49,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c4fa755565f449d7bf00513882e91064 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/c4fa755565f449d7bf00513882e91064 2024-11-14T07:16:49,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f70461ce012b40c9bbd72f4449ad371c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/f70461ce012b40c9bbd72f4449ad371c 2024-11-14T07:16:49,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/58b1e70503fd4314977aa52a04be036d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/58b1e70503fd4314977aa52a04be036d 2024-11-14T07:16:49,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/081132b961f8457eafe21a55d356fabf to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/081132b961f8457eafe21a55d356fabf 2024-11-14T07:16:49,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/02ef130af03c4fbeaf52debe4f3cb652 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/02ef130af03c4fbeaf52debe4f3cb652 2024-11-14T07:16:49,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/88ba138345e346c8b7a6a32d6246be7c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/88ba138345e346c8b7a6a32d6246be7c 2024-11-14T07:16:49,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/a99c6933caaa4b44b2fbc7c0bc27680f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/a99c6933caaa4b44b2fbc7c0bc27680f 2024-11-14T07:16:49,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/1ec31250d202409eb2821303251ed056 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/1ec31250d202409eb2821303251ed056 2024-11-14T07:16:49,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/48ad15a121b94f1aa21611bc68414c25 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/48ad15a121b94f1aa21611bc68414c25 2024-11-14T07:16:49,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/2e907aee9c2242ce95e2d528ef0c5c05 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/2e907aee9c2242ce95e2d528ef0c5c05 2024-11-14T07:16:49,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/51e0f6bc1a1040468a26267681db2d79 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/51e0f6bc1a1040468a26267681db2d79 2024-11-14T07:16:49,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7b5f413f4dd64ca2ac9d1c443dfa298b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/b9733341dd294eb2b3503cbae6700eaf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4bd337a5723d4d179debc5f9870a1a26, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/8e1d3bbd36254fb99092362c565a7f9a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4fb6c43a3d0d475697646e9f77c83c8c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/fd8f363c51514ab6934c11c1bbe53bb7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7bd8628162bf4a358468f9f9682201f3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/567b07f6889d4b6f8f9d01e8278bbc70, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1f7da0cac0a4296b8468af20c7ea0c5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7eb01b50a0964b2aaef1c89454a3f1f4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/2f6d22ac3f88449e99d4be31eeb5c84d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1a9c5186aa1492c8079fe6c1b8f1068, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/41e1e08060ad475aa5b90eaa12421c67, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5f8c2b0661d64ae5b9d950c379c3d7a9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/263b430d84a746fcae213f8d893374b4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f69cc867e7ef41d3a724c54d3fe25a02, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6590eb4be77142cdba96b0c2dc541004, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/350340ac8b604b399686e872cfb150ae, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/530c68b40eed4afc81cf10682f3d6147, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/e5cb4bdfeb8d440cbfc37842c3f5db5b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6a2a7859f09f43b1b8f0b7f51481b199, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ffae1641892841bfb790b12a632454ba, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5415320a2d2f4372b35f56e108cf7d50, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/99c849c65f71492498c827c70e47eb59, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/de07229648564393984c027b81b3b8f4] to archive 2024-11-14T07:16:49,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:16:49,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7b5f413f4dd64ca2ac9d1c443dfa298b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7b5f413f4dd64ca2ac9d1c443dfa298b 2024-11-14T07:16:49,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/b9733341dd294eb2b3503cbae6700eaf to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/b9733341dd294eb2b3503cbae6700eaf 2024-11-14T07:16:49,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4bd337a5723d4d179debc5f9870a1a26 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4bd337a5723d4d179debc5f9870a1a26 2024-11-14T07:16:49,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/8e1d3bbd36254fb99092362c565a7f9a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/8e1d3bbd36254fb99092362c565a7f9a 2024-11-14T07:16:49,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4fb6c43a3d0d475697646e9f77c83c8c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/4fb6c43a3d0d475697646e9f77c83c8c 2024-11-14T07:16:49,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/fd8f363c51514ab6934c11c1bbe53bb7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/fd8f363c51514ab6934c11c1bbe53bb7 2024-11-14T07:16:49,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7bd8628162bf4a358468f9f9682201f3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7bd8628162bf4a358468f9f9682201f3 2024-11-14T07:16:49,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/567b07f6889d4b6f8f9d01e8278bbc70 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/567b07f6889d4b6f8f9d01e8278bbc70 2024-11-14T07:16:49,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1f7da0cac0a4296b8468af20c7ea0c5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1f7da0cac0a4296b8468af20c7ea0c5 2024-11-14T07:16:49,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7eb01b50a0964b2aaef1c89454a3f1f4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/7eb01b50a0964b2aaef1c89454a3f1f4 2024-11-14T07:16:49,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/2f6d22ac3f88449e99d4be31eeb5c84d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/2f6d22ac3f88449e99d4be31eeb5c84d 2024-11-14T07:16:49,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1a9c5186aa1492c8079fe6c1b8f1068 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/c1a9c5186aa1492c8079fe6c1b8f1068 2024-11-14T07:16:49,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/41e1e08060ad475aa5b90eaa12421c67 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/41e1e08060ad475aa5b90eaa12421c67 2024-11-14T07:16:49,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5f8c2b0661d64ae5b9d950c379c3d7a9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5f8c2b0661d64ae5b9d950c379c3d7a9 2024-11-14T07:16:49,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/263b430d84a746fcae213f8d893374b4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/263b430d84a746fcae213f8d893374b4 2024-11-14T07:16:49,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f69cc867e7ef41d3a724c54d3fe25a02 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f69cc867e7ef41d3a724c54d3fe25a02 2024-11-14T07:16:49,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6590eb4be77142cdba96b0c2dc541004 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6590eb4be77142cdba96b0c2dc541004 2024-11-14T07:16:49,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/350340ac8b604b399686e872cfb150ae to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/350340ac8b604b399686e872cfb150ae 2024-11-14T07:16:49,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/530c68b40eed4afc81cf10682f3d6147 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/530c68b40eed4afc81cf10682f3d6147 2024-11-14T07:16:49,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/e5cb4bdfeb8d440cbfc37842c3f5db5b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/e5cb4bdfeb8d440cbfc37842c3f5db5b 2024-11-14T07:16:49,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6a2a7859f09f43b1b8f0b7f51481b199 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/6a2a7859f09f43b1b8f0b7f51481b199 2024-11-14T07:16:49,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ffae1641892841bfb790b12a632454ba to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ffae1641892841bfb790b12a632454ba 2024-11-14T07:16:49,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5415320a2d2f4372b35f56e108cf7d50 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/5415320a2d2f4372b35f56e108cf7d50 2024-11-14T07:16:49,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/99c849c65f71492498c827c70e47eb59 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/99c849c65f71492498c827c70e47eb59 2024-11-14T07:16:49,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/de07229648564393984c027b81b3b8f4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/de07229648564393984c027b81b3b8f4 2024-11-14T07:16:49,985 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=4 2024-11-14T07:16:49,987 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. 2024-11-14T07:16:49,987 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 26fca7a5cd511ec9993090c4fc5431d4: Waiting for close lock at 1731568608233Running coprocessor pre-close hooks at 1731568608233Disabling compacts and flushes for region at 1731568608233Disabling writes for close at 1731568608233Obtaining lock to block concurrent updates at 1731568608233Preparing flush snapshotting stores in 26fca7a5cd511ec9993090c4fc5431d4 at 1731568608233Finished memstore snapshotting TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731568608233Flushing stores of TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4. at 1731568608234 (+1 ms)Flushing 26fca7a5cd511ec9993090c4fc5431d4/A: creating writer at 1731568608234Flushing mob file 26fca7a5cd511ec9993090c4fc5431d4/A: appending metadata at 1731568608239 (+5 ms)Flushing mob file 26fca7a5cd511ec9993090c4fc5431d4/A: closing flushed file at 1731568608239Flushing 26fca7a5cd511ec9993090c4fc5431d4/A: appending metadata at 1731568608655 (+416 ms)Flushing 26fca7a5cd511ec9993090c4fc5431d4/A: closing flushed file at 1731568608655Flushing 26fca7a5cd511ec9993090c4fc5431d4/B: creating writer at 1731568609065 (+410 ms)Flushing 26fca7a5cd511ec9993090c4fc5431d4/B: appending metadata at 1731568609068 (+3 ms)Flushing 26fca7a5cd511ec9993090c4fc5431d4/B: closing flushed file at 1731568609068Flushing 26fca7a5cd511ec9993090c4fc5431d4/C: creating writer at 1731568609480 (+412 ms)Flushing 26fca7a5cd511ec9993090c4fc5431d4/C: appending metadata at 1731568609485 (+5 ms)Flushing 26fca7a5cd511ec9993090c4fc5431d4/C: closing flushed file at 1731568609485Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2df62877: reopening flushed file at 1731568609898 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b4fb676: reopening flushed file at 1731568609904 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ae92de: reopening flushed file at 1731568609908 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 26fca7a5cd511ec9993090c4fc5431d4 in 1679ms, sequenceid=421, compaction requested=true at 1731568609912 (+4 ms)Writing region close event to WAL at 1731568609982 (+70 ms)Running coprocessor post-close hooks at 1731568609986 (+4 ms)Closed at 1731568609987 (+1 ms) 2024-11-14T07:16:49,989 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:49,990 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=26fca7a5cd511ec9993090c4fc5431d4, regionState=CLOSED 2024-11-14T07:16:49,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:16:49,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T07:16:49,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 26fca7a5cd511ec9993090c4fc5431d4, server=c7aa6e2790b3,33845,1731568550163 in 1.9130 sec 2024-11-14T07:16:49,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-14T07:16:49,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26fca7a5cd511ec9993090c4fc5431d4, UNASSIGN in 1.9190 sec 2024-11-14T07:16:49,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T07:16:49,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9210 sec 2024-11-14T07:16:49,999 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568609999"}]},"ts":"1731568609999"} 2024-11-14T07:16:50,001 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:16:50,001 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:16:50,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9350 sec 2024-11-14T07:16:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:16:50,222 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:50,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:16:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:50,224 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:50,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:16:50,224 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:50,227 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,229 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/recovered.edits] 2024-11-14T07:16:50,232 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/515ee20ded44407fa6b873385a07d03c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/515ee20ded44407fa6b873385a07d03c 2024-11-14T07:16:50,233 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5201c4f574ae42018b8c9ef0dd2ce5c6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/5201c4f574ae42018b8c9ef0dd2ce5c6 2024-11-14T07:16:50,234 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/f2e0cf1a1e1f43e299a47bdb80cf7845 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/A/f2e0cf1a1e1f43e299a47bdb80cf7845 2024-11-14T07:16:50,236 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/39c4d62235214ed1ab61da9ccffc3fb5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/39c4d62235214ed1ab61da9ccffc3fb5 2024-11-14T07:16:50,238 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/409b6b419e7843a8bfdb9b5e82a1fda9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/409b6b419e7843a8bfdb9b5e82a1fda9 2024-11-14T07:16:50,239 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b78f9e17dc4e4bbbb78782736170580d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/B/b78f9e17dc4e4bbbb78782736170580d 2024-11-14T07:16:50,241 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/3da31818e16d4cc38074b4102b4047bc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/3da31818e16d4cc38074b4102b4047bc 2024-11-14T07:16:50,242 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ef6cde27edec43e298b50a9fe4012b1c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/ef6cde27edec43e298b50a9fe4012b1c 2024-11-14T07:16:50,244 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f932ae93a687493b8113c12fc106f82c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/C/f932ae93a687493b8113c12fc106f82c 2024-11-14T07:16:50,246 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/recovered.edits/424.seqid to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4/recovered.edits/424.seqid 2024-11-14T07:16:50,247 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,247 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:16:50,247 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:16:50,248 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T07:16:50,251 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141188c96c9279483fbce744cf49cf88be_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141188c96c9279483fbce744cf49cf88be_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,252 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114137e6958608745e7979558a434b53860_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114137e6958608745e7979558a434b53860_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,252 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111425e57965c7f94c498b51557dd4fdc61c_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111425e57965c7f94c498b51557dd4fdc61c_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,253 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142acce97db4684f32b6acf32d6a03789c_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142acce97db4684f32b6acf32d6a03789c_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,254 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143279e02969714954a64e5e4501fd979f_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143279e02969714954a64e5e4501fd979f_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,255 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114476ffc8a5cf34c4dbc43914c0191eafd_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114476ffc8a5cf34c4dbc43914c0191eafd_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,256 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114590cca067294472bbf07044eabece4b1_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114590cca067294472bbf07044eabece4b1_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,257 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ec27ededf0d4649a25ad1f275c4a6f0_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ec27ededf0d4649a25ad1f275c4a6f0_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,258 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111463c88de3c0434ad3946c3c86b39848f2_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111463c88de3c0434ad3946c3c86b39848f2_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,259 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111485cd016060a54c589e2c240271dd39fb_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111485cd016060a54c589e2c240271dd39fb_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,260 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494631af17dae4b349c8ecf0c6900f7e0_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494631af17dae4b349c8ecf0c6900f7e0_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,261 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114965c02c763c6424aa31564973f07ace5_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114965c02c763c6424aa31564973f07ace5_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,262 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aae1bd8a7cbb4ac589a9b6b493d31bf3_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aae1bd8a7cbb4ac589a9b6b493d31bf3_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,264 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ab0a3cebc22e454cbfa6115a568c5048_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ab0a3cebc22e454cbfa6115a568c5048_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,265 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b782d613a3c74130b661f55dedd93354_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b782d613a3c74130b661f55dedd93354_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,265 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b8faff48c31a49738cbe9726ac3fedb7_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b8faff48c31a49738cbe9726ac3fedb7_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,266 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c39510b7e6ab40da851097075293499f_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c39510b7e6ab40da851097075293499f_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,267 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3f98aa3db7640c897f521e0ff625f49_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3f98aa3db7640c897f521e0ff625f49_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,268 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d252e37077c44917bbabbb987a05fcfd_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d252e37077c44917bbabbb987a05fcfd_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,269 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e75029505a5c4f0eb79cc331f1971136_26fca7a5cd511ec9993090c4fc5431d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e75029505a5c4f0eb79cc331f1971136_26fca7a5cd511ec9993090c4fc5431d4 2024-11-14T07:16:50,269 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:16:50,271 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:50,273 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:16:50,275 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:16:50,277 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:50,277 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:16:50,277 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731568610277"}]},"ts":"9223372036854775807"} 2024-11-14T07:16:50,280 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:16:50,280 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 26fca7a5cd511ec9993090c4fc5431d4, NAME => 'TestAcidGuarantees,,1731568581722.26fca7a5cd511ec9993090c4fc5431d4.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:16:50,280 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:16:50,280 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731568610280"}]},"ts":"9223372036854775807"} 2024-11-14T07:16:50,282 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:16:50,283 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:50,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-14T07:16:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:16:50,332 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:50,343 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 250) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1006931501_22 at /127.0.0.1:58434 [Waiting for operation #452] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-406272114_22 at /127.0.0.1:34334 [Waiting for operation #220] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1006931501_22 at /127.0.0.1:48522 [Waiting for operation #667] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 443) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5471 (was 5862) 2024-11-14T07:16:50,352 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=5471 2024-11-14T07:16:50,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:16:50,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:16:50,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:16:50,355 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:16:50,355 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:50,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-14T07:16:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T07:16:50,356 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:16:50,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742051_1227 (size=1061) 2024-11-14T07:16:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T07:16:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T07:16:50,765 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:16:50,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742052_1228 (size=53) 2024-11-14T07:16:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T07:16:51,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:16:51,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b5566aea29a31d45dee688596a38fef4, disabling compactions & flushes 2024-11-14T07:16:51,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,177 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,177 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. after waiting 0 ms 2024-11-14T07:16:51,177 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,177 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,177 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b5566aea29a31d45dee688596a38fef4: Waiting for close lock at 1731568611176Disabling compacts and flushes for region at 1731568611176Disabling writes for close at 1731568611177 (+1 ms)Writing region close event to WAL at 1731568611177Closed at 1731568611177 2024-11-14T07:16:51,179 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:16:51,180 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731568611179"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731568611179"}]},"ts":"1731568611179"} 2024-11-14T07:16:51,183 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:16:51,184 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:16:51,184 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568611184"}]},"ts":"1731568611184"} 2024-11-14T07:16:51,187 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:16:51,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, ASSIGN}] 2024-11-14T07:16:51,189 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, ASSIGN 2024-11-14T07:16:51,189 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=false 2024-11-14T07:16:51,340 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=b5566aea29a31d45dee688596a38fef4, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:51,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, ASSIGN because future has completed 2024-11-14T07:16:51,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T07:16:51,501 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,501 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:16:51,501 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,502 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:16:51,502 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,502 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,506 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,507 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:51,507 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566aea29a31d45dee688596a38fef4 columnFamilyName A 2024-11-14T07:16:51,507 DEBUG [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:51,508 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.HStore(327): Store=b5566aea29a31d45dee688596a38fef4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:51,508 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,509 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:51,509 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566aea29a31d45dee688596a38fef4 columnFamilyName B 2024-11-14T07:16:51,509 DEBUG [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:51,509 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.HStore(327): Store=b5566aea29a31d45dee688596a38fef4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:51,509 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,510 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:16:51,511 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566aea29a31d45dee688596a38fef4 columnFamilyName C 2024-11-14T07:16:51,511 DEBUG [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:16:51,511 INFO [StoreOpener-b5566aea29a31d45dee688596a38fef4-1 {}] regionserver.HStore(327): Store=b5566aea29a31d45dee688596a38fef4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:16:51,511 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,511 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,513 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,513 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,514 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:16:51,515 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,517 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:16:51,517 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened b5566aea29a31d45dee688596a38fef4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64520308, jitterRate=-0.038572490215301514}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:16:51,517 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:51,518 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for b5566aea29a31d45dee688596a38fef4: Running coprocessor pre-open hook at 1731568611502Writing region info on filesystem at 1731568611502Initializing all the Stores at 1731568611505 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568611505Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568611506 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568611506Cleaning up temporary data from old regions at 1731568611513 (+7 ms)Running coprocessor post-open hooks at 1731568611517 (+4 ms)Region opened successfully at 1731568611517 2024-11-14T07:16:51,518 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., pid=61, masterSystemTime=1731568611498 2024-11-14T07:16:51,520 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,520 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:51,520 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=b5566aea29a31d45dee688596a38fef4, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:51,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:16:51,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T07:16:51,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 in 178 msec 2024-11-14T07:16:51,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-14T07:16:51,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, ASSIGN in 337 msec 2024-11-14T07:16:51,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:16:51,526 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568611526"}]},"ts":"1731568611526"} 2024-11-14T07:16:51,527 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:16:51,528 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:16:51,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-14T07:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T07:16:52,502 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:52,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,505 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b752b4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,506 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405509c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,510 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:16:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,37073,1731568549456 2024-11-14T07:16:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c4a80f0 2024-11-14T07:16:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:16:52,512 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:16:52,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,514 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576d3475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad10086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f1c75fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,519 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b334c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,520 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b306dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398d6aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,523 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bca325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,525 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562fb9d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,527 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e99d711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,528 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fab7725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,530 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1481c5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259fe7a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,533 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ff729f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ebeca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,536 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4cbc79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efcff6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,540 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cab4ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,541 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e436efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:16:52,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:16:52,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:16:52,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:16:52,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:16:52,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7244b721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:16:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:16:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:16:52,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:16:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2443427a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:16:52,547 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,547 DEBUG [Thread-1078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,548 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,548 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,548 DEBUG [Thread-1087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,550 DEBUG [Thread-1072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,550 DEBUG [Thread-1085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-14T07:16:52,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T07:16:52,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,552 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,554 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,564 DEBUG [Thread-1070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,564 DEBUG [Thread-1074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,566 DEBUG [Thread-1076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:16:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:52,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:52,576 DEBUG [Thread-1076 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:16:52,576 DEBUG [Thread-1076 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:16:52,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:16:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:16:52,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/53043f98824e4d52aeba1ef739d523c3 is 50, key is test_row_0/A:col10/1731568612563/Put/seqid=0 2024-11-14T07:16:52,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568612591, max timestamp=1731568612606Num uniques 859; 2024-11-14T07:16:52,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568612591, max timestamp=1731568612606Num uniques 859; 2024-11-14T07:16:52,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568612591, max timestamp=1731568612606Num uniques 859; 2024-11-14T07:16:52,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:52,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:52,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:52,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742053_1229 (size=14341) 2024-11-14T07:16:52,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/53043f98824e4d52aeba1ef739d523c3 2024-11-14T07:16:52,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568672632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568672634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568672635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568672637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568672638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T07:16:52,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/f7e4e215150c47dc96aecc6c8ca06186 is 50, key is test_row_0/B:col10/1731568612563/Put/seqid=0 2024-11-14T07:16:52,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742054_1230 (size=12001) 2024-11-14T07:16:52,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T07:16:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:52,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568672752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568672752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568672752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568672752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568672752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T07:16:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:52,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T07:16:52,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568672972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568672972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568672972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568672972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568672972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T07:16:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/f7e4e215150c47dc96aecc6c8ca06186 2024-11-14T07:16:53,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/6133c6df93dd4a4dbc456e9988959d2d is 50, key is test_row_0/C:col10/1731568612563/Put/seqid=0 2024-11-14T07:16:53,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742055_1231 (size=12001) 2024-11-14T07:16:53,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T07:16:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T07:16:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568673281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568673282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568673282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568673282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568673282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T07:16:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T07:16:53,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:53,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/6133c6df93dd4a4dbc456e9988959d2d 2024-11-14T07:16:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/53043f98824e4d52aeba1ef739d523c3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/53043f98824e4d52aeba1ef739d523c3 2024-11-14T07:16:53,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/53043f98824e4d52aeba1ef739d523c3, entries=200, sequenceid=11, filesize=14.0 K 2024-11-14T07:16:53,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/f7e4e215150c47dc96aecc6c8ca06186 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/f7e4e215150c47dc96aecc6c8ca06186 2024-11-14T07:16:53,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/f7e4e215150c47dc96aecc6c8ca06186, entries=150, sequenceid=11, filesize=11.7 K 2024-11-14T07:16:53,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/6133c6df93dd4a4dbc456e9988959d2d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/6133c6df93dd4a4dbc456e9988959d2d 2024-11-14T07:16:53,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/6133c6df93dd4a4dbc456e9988959d2d, entries=150, sequenceid=11, filesize=11.7 K 2024-11-14T07:16:53,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b5566aea29a31d45dee688596a38fef4 in 989ms, sequenceid=11, compaction requested=false 2024-11-14T07:16:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:53,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T07:16:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:53,627 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T07:16:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:16:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/676b5071f95c457a9169e8cc944f37c7 is 50, key is test_row_0/A:col10/1731568612631/Put/seqid=0 2024-11-14T07:16:53,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742056_1232 (size=12001) 2024-11-14T07:16:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T07:16:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:53,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568673796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568673796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568673797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568673800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568673801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568673911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568673911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568673912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568673912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568673912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,046 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/676b5071f95c457a9169e8cc944f37c7 2024-11-14T07:16:54,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/a59b175384d345cbb442faac190790c1 is 50, key is test_row_0/B:col10/1731568612631/Put/seqid=0 2024-11-14T07:16:54,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742057_1233 (size=12001) 2024-11-14T07:16:54,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568674122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568674122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568674122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568674122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568674122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568674431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568674431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568674432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568674432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568674432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:54,484 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/a59b175384d345cbb442faac190790c1 2024-11-14T07:16:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/5a2c2eff1ee9447e8c84cfef9a80b28c is 50, key is test_row_0/C:col10/1731568612631/Put/seqid=0 2024-11-14T07:16:54,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742058_1234 (size=12001) 2024-11-14T07:16:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T07:16:54,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/5a2c2eff1ee9447e8c84cfef9a80b28c 2024-11-14T07:16:54,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/676b5071f95c457a9169e8cc944f37c7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/676b5071f95c457a9169e8cc944f37c7 2024-11-14T07:16:54,911 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/676b5071f95c457a9169e8cc944f37c7, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T07:16:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/a59b175384d345cbb442faac190790c1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a59b175384d345cbb442faac190790c1 2024-11-14T07:16:54,918 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a59b175384d345cbb442faac190790c1, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T07:16:54,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/5a2c2eff1ee9447e8c84cfef9a80b28c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5a2c2eff1ee9447e8c84cfef9a80b28c 2024-11-14T07:16:54,926 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5a2c2eff1ee9447e8c84cfef9a80b28c, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T07:16:54,927 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for b5566aea29a31d45dee688596a38fef4 in 1299ms, sequenceid=47, compaction requested=false 2024-11-14T07:16:54,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:54,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:54,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T07:16:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T07:16:54,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T07:16:54,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3740 sec 2024-11-14T07:16:54,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.3830 sec 2024-11-14T07:16:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:54,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:16:54,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:54,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:54,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:54,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:54,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:54,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:54,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f9e013fef57d4b059ab93c6e1a28dbb0 is 50, key is test_row_0/A:col10/1731568614944/Put/seqid=0 2024-11-14T07:16:54,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742059_1235 (size=12001) 2024-11-14T07:16:54,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f9e013fef57d4b059ab93c6e1a28dbb0 2024-11-14T07:16:54,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/d31e123189794e0799bf48bb77dac293 is 50, key is test_row_0/B:col10/1731568614944/Put/seqid=0 2024-11-14T07:16:55,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742060_1236 (size=12001) 2024-11-14T07:16:55,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/d31e123189794e0799bf48bb77dac293 2024-11-14T07:16:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568674988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/0674780fb3ce4852bc2080344c9be868 is 50, key is test_row_0/C:col10/1731568614944/Put/seqid=0 2024-11-14T07:16:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568675023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568675024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568675024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568675025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742061_1237 (size=12001) 2024-11-14T07:16:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568675131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568675142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568675142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568675142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568675142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568675342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568675352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568675352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568675352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568675352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/0674780fb3ce4852bc2080344c9be868 2024-11-14T07:16:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f9e013fef57d4b059ab93c6e1a28dbb0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f9e013fef57d4b059ab93c6e1a28dbb0 2024-11-14T07:16:55,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f9e013fef57d4b059ab93c6e1a28dbb0, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T07:16:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-14T07:16:55,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/d31e123189794e0799bf48bb77dac293 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/d31e123189794e0799bf48bb77dac293 2024-11-14T07:16:55,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/d31e123189794e0799bf48bb77dac293, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T07:16:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/0674780fb3ce4852bc2080344c9be868 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/0674780fb3ce4852bc2080344c9be868 2024-11-14T07:16:55,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/0674780fb3ce4852bc2080344c9be868, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T07:16:55,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b5566aea29a31d45dee688596a38fef4 in 551ms, sequenceid=60, compaction requested=true 2024-11-14T07:16:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:55,496 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:55,496 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:55,497 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:55,497 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:55,497 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:16:55,497 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:16:55,498 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:55,498 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:55,498 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/53043f98824e4d52aeba1ef739d523c3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/676b5071f95c457a9169e8cc944f37c7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f9e013fef57d4b059ab93c6e1a28dbb0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=37.4 K 2024-11-14T07:16:55,498 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/f7e4e215150c47dc96aecc6c8ca06186, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a59b175384d345cbb442faac190790c1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/d31e123189794e0799bf48bb77dac293] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=35.2 K 2024-11-14T07:16:55,498 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e4e215150c47dc96aecc6c8ca06186, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731568612563 2024-11-14T07:16:55,498 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53043f98824e4d52aeba1ef739d523c3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731568612561 2024-11-14T07:16:55,498 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 676b5071f95c457a9169e8cc944f37c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568612626 2024-11-14T07:16:55,498 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a59b175384d345cbb442faac190790c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568612626 2024-11-14T07:16:55,499 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9e013fef57d4b059ab93c6e1a28dbb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731568613796 2024-11-14T07:16:55,499 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d31e123189794e0799bf48bb77dac293, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731568613796 2024-11-14T07:16:55,506 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:55,506 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/8e6ca48c463c4f86bcd4405315024391 is 50, key is test_row_0/B:col10/1731568614944/Put/seqid=0 2024-11-14T07:16:55,508 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:55,509 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/5db818b8107f496b8a1937a1dc48496f is 50, key is test_row_0/A:col10/1731568614944/Put/seqid=0 2024-11-14T07:16:55,515 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:16:55,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742062_1238 (size=12104) 2024-11-14T07:16:55,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742063_1239 (size=12104) 2024-11-14T07:16:55,558 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/5db818b8107f496b8a1937a1dc48496f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/5db818b8107f496b8a1937a1dc48496f 2024-11-14T07:16:55,567 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into 5db818b8107f496b8a1937a1dc48496f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:55,567 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:55,567 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=13, startTime=1731568615496; duration=0sec 2024-11-14T07:16:55,567 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:55,567 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:16:55,567 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:55,569 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:55,569 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:16:55,569 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:55,570 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/6133c6df93dd4a4dbc456e9988959d2d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5a2c2eff1ee9447e8c84cfef9a80b28c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/0674780fb3ce4852bc2080344c9be868] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=35.2 K 2024-11-14T07:16:55,571 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6133c6df93dd4a4dbc456e9988959d2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731568612563 2024-11-14T07:16:55,571 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a2c2eff1ee9447e8c84cfef9a80b28c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568612626 2024-11-14T07:16:55,572 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0674780fb3ce4852bc2080344c9be868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731568613796 2024-11-14T07:16:55,581 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:55,582 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2975148177414ee8aa3c5c469170bcb5 is 50, key is test_row_0/C:col10/1731568614944/Put/seqid=0 2024-11-14T07:16:55,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742064_1240 (size=12104) 2024-11-14T07:16:55,612 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2975148177414ee8aa3c5c469170bcb5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2975148177414ee8aa3c5c469170bcb5 2024-11-14T07:16:55,618 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into 2975148177414ee8aa3c5c469170bcb5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:55,618 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:55,618 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=13, startTime=1731568615497; duration=0sec 2024-11-14T07:16:55,618 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:55,618 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:55,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:16:55,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:55,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:55,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:55,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:55,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:55,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:55,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/47b900ff02964c559e5b4dffa47826f9 is 50, key is test_row_0/A:col10/1731568615654/Put/seqid=0 2024-11-14T07:16:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568675670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568675671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568675672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742065_1241 (size=14341) 2024-11-14T07:16:55,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/47b900ff02964c559e5b4dffa47826f9 2024-11-14T07:16:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568675679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568675678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/a000211650db491b9ee1338be07dfeca is 50, key is test_row_0/B:col10/1731568615654/Put/seqid=0 2024-11-14T07:16:55,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742066_1242 (size=12001) 2024-11-14T07:16:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568675781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568675782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568675782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568675792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568675792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,927 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/8e6ca48c463c4f86bcd4405315024391 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8e6ca48c463c4f86bcd4405315024391 2024-11-14T07:16:55,931 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into 8e6ca48c463c4f86bcd4405315024391(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:55,931 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:55,931 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=13, startTime=1731568615496; duration=0sec 2024-11-14T07:16:55,931 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:55,931 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:16:55,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568675991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568675992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:55,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568675992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568676002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568676002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/a000211650db491b9ee1338be07dfeca 2024-11-14T07:16:56,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/387f69eff9e8421d9dfa12933bfd6551 is 50, key is test_row_0/C:col10/1731568615654/Put/seqid=0 2024-11-14T07:16:56,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742067_1243 (size=12001) 2024-11-14T07:16:56,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/387f69eff9e8421d9dfa12933bfd6551 2024-11-14T07:16:56,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/47b900ff02964c559e5b4dffa47826f9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/47b900ff02964c559e5b4dffa47826f9 2024-11-14T07:16:56,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/47b900ff02964c559e5b4dffa47826f9, entries=200, sequenceid=82, filesize=14.0 K 2024-11-14T07:16:56,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/a000211650db491b9ee1338be07dfeca as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a000211650db491b9ee1338be07dfeca 2024-11-14T07:16:56,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a000211650db491b9ee1338be07dfeca, entries=150, sequenceid=82, filesize=11.7 K 2024-11-14T07:16:56,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/387f69eff9e8421d9dfa12933bfd6551 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/387f69eff9e8421d9dfa12933bfd6551 2024-11-14T07:16:56,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/387f69eff9e8421d9dfa12933bfd6551, entries=150, sequenceid=82, filesize=11.7 K 2024-11-14T07:16:56,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for b5566aea29a31d45dee688596a38fef4 in 513ms, sequenceid=82, compaction requested=false 2024-11-14T07:16:56,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:56,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T07:16:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:56,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f07c5c547d6d4c28b2c25d7bf4a91739 is 50, key is test_row_0/A:col10/1731568616303/Put/seqid=0 2024-11-14T07:16:56,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742068_1244 (size=14341) 2024-11-14T07:16:56,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f07c5c547d6d4c28b2c25d7bf4a91739 2024-11-14T07:16:56,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/c0afffd0f0d949ec89e2a101502b4a4b is 50, key is test_row_0/B:col10/1731568616303/Put/seqid=0 2024-11-14T07:16:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568676321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568676322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568676323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568676328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568676329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742069_1245 (size=12001) 2024-11-14T07:16:56,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/c0afffd0f0d949ec89e2a101502b4a4b 2024-11-14T07:16:56,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/c2755e908bdc4173aefaedd14dfa7b73 is 50, key is test_row_0/C:col10/1731568616303/Put/seqid=0 2024-11-14T07:16:56,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742070_1246 (size=12001) 2024-11-14T07:16:56,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568676432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568676432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568676432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568676442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568676442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568676642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568676641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568676642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568676652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568676652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T07:16:56,712 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:56,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T07:16:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T07:16:56,714 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:56,714 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:56,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:56,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/c2755e908bdc4173aefaedd14dfa7b73 2024-11-14T07:16:56,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f07c5c547d6d4c28b2c25d7bf4a91739 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07c5c547d6d4c28b2c25d7bf4a91739 2024-11-14T07:16:56,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07c5c547d6d4c28b2c25d7bf4a91739, entries=200, sequenceid=99, filesize=14.0 K 2024-11-14T07:16:56,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/c0afffd0f0d949ec89e2a101502b4a4b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/c0afffd0f0d949ec89e2a101502b4a4b 2024-11-14T07:16:56,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/c0afffd0f0d949ec89e2a101502b4a4b, entries=150, sequenceid=99, filesize=11.7 K 2024-11-14T07:16:56,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/c2755e908bdc4173aefaedd14dfa7b73 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/c2755e908bdc4173aefaedd14dfa7b73 2024-11-14T07:16:56,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/c2755e908bdc4173aefaedd14dfa7b73, entries=150, sequenceid=99, filesize=11.7 K 2024-11-14T07:16:56,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for b5566aea29a31d45dee688596a38fef4 in 463ms, sequenceid=99, compaction requested=true 2024-11-14T07:16:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:16:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:16:56,767 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:56,767 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:16:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:56,769 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:56,769 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:56,769 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:16:56,769 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:16:56,769 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:56,769 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8e6ca48c463c4f86bcd4405315024391, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a000211650db491b9ee1338be07dfeca, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/c0afffd0f0d949ec89e2a101502b4a4b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=35.3 K 2024-11-14T07:16:56,769 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:56,769 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/5db818b8107f496b8a1937a1dc48496f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/47b900ff02964c559e5b4dffa47826f9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07c5c547d6d4c28b2c25d7bf4a91739] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=39.8 K 2024-11-14T07:16:56,770 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db818b8107f496b8a1937a1dc48496f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731568613796 2024-11-14T07:16:56,770 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6ca48c463c4f86bcd4405315024391, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731568613796 2024-11-14T07:16:56,770 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a000211650db491b9ee1338be07dfeca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731568614974 2024-11-14T07:16:56,770 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47b900ff02964c559e5b4dffa47826f9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731568614974 2024-11-14T07:16:56,771 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c0afffd0f0d949ec89e2a101502b4a4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731568615670 2024-11-14T07:16:56,771 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f07c5c547d6d4c28b2c25d7bf4a91739, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731568615670 2024-11-14T07:16:56,778 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:56,779 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/eaf43deb11454edb9885e314cca2b622 is 50, key is test_row_0/A:col10/1731568616303/Put/seqid=0 2024-11-14T07:16:56,781 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:56,782 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/6820a7ac0ce94b51b2fdba93ec5ad735 is 50, key is test_row_0/B:col10/1731568616303/Put/seqid=0 2024-11-14T07:16:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742072_1248 (size=12207) 2024-11-14T07:16:56,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742071_1247 (size=12207) 2024-11-14T07:16:56,800 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/6820a7ac0ce94b51b2fdba93ec5ad735 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6820a7ac0ce94b51b2fdba93ec5ad735 2024-11-14T07:16:56,805 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into 6820a7ac0ce94b51b2fdba93ec5ad735(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:56,805 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:56,806 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=13, startTime=1731568616767; duration=0sec 2024-11-14T07:16:56,806 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:16:56,806 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:16:56,806 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:16:56,807 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:16:56,807 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:16:56,807 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:56,807 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2975148177414ee8aa3c5c469170bcb5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/387f69eff9e8421d9dfa12933bfd6551, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/c2755e908bdc4173aefaedd14dfa7b73] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=35.3 K 2024-11-14T07:16:56,807 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2975148177414ee8aa3c5c469170bcb5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731568613796 2024-11-14T07:16:56,808 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 387f69eff9e8421d9dfa12933bfd6551, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731568614974 2024-11-14T07:16:56,808 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c2755e908bdc4173aefaedd14dfa7b73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731568615670 2024-11-14T07:16:56,816 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:16:56,817 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/00c8b614b3b44e58b80dee81579e7c72 is 50, key is test_row_0/C:col10/1731568616303/Put/seqid=0 2024-11-14T07:16:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T07:16:56,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742073_1249 (size=12207) 2024-11-14T07:16:56,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T07:16:56,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:56,866 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T07:16:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:56,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/0399afccdea34d75b6e55a69b1f35e9c is 50, key is test_row_0/A:col10/1731568616326/Put/seqid=0 2024-11-14T07:16:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742074_1250 (size=12001) 2024-11-14T07:16:56,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/0399afccdea34d75b6e55a69b1f35e9c 2024-11-14T07:16:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/fc033a1ecd034c0fb357f52da18fcbf8 is 50, key is test_row_0/B:col10/1731568616326/Put/seqid=0 2024-11-14T07:16:56,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742075_1251 (size=12001) 2024-11-14T07:16:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:56,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:56,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568676968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568676969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568676969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568676970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568676974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T07:16:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568677082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568677082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568677082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568677083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568677092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,199 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/eaf43deb11454edb9885e314cca2b622 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/eaf43deb11454edb9885e314cca2b622 2024-11-14T07:16:57,203 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into eaf43deb11454edb9885e314cca2b622(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:57,203 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:57,203 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=13, startTime=1731568616767; duration=0sec 2024-11-14T07:16:57,204 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:57,204 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:16:57,226 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/00c8b614b3b44e58b80dee81579e7c72 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/00c8b614b3b44e58b80dee81579e7c72 2024-11-14T07:16:57,231 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into 00c8b614b3b44e58b80dee81579e7c72(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:16:57,231 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:57,231 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=13, startTime=1731568616767; duration=0sec 2024-11-14T07:16:57,231 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:16:57,231 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:16:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568677291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568677292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568677292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568677292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/fc033a1ecd034c0fb357f52da18fcbf8 2024-11-14T07:16:57,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568677302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e271cf630e2441b3b71aa4f7d249ec2e is 50, key is test_row_0/C:col10/1731568616326/Put/seqid=0 2024-11-14T07:16:57,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742076_1252 (size=12001) 2024-11-14T07:16:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T07:16:57,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568677601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568677601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568677601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568677602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568677612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:57,714 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e271cf630e2441b3b71aa4f7d249ec2e 2024-11-14T07:16:57,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/0399afccdea34d75b6e55a69b1f35e9c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/0399afccdea34d75b6e55a69b1f35e9c 2024-11-14T07:16:57,722 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/0399afccdea34d75b6e55a69b1f35e9c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T07:16:57,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/fc033a1ecd034c0fb357f52da18fcbf8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/fc033a1ecd034c0fb357f52da18fcbf8 2024-11-14T07:16:57,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/fc033a1ecd034c0fb357f52da18fcbf8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T07:16:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e271cf630e2441b3b71aa4f7d249ec2e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e271cf630e2441b3b71aa4f7d249ec2e 2024-11-14T07:16:57,730 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e271cf630e2441b3b71aa4f7d249ec2e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T07:16:57,731 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b5566aea29a31d45dee688596a38fef4 in 865ms, sequenceid=117, compaction requested=false 2024-11-14T07:16:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T07:16:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T07:16:57,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T07:16:57,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-11-14T07:16:57,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.0230 sec 2024-11-14T07:16:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T07:16:57,862 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:57,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T07:16:57,864 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T07:16:57,864 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:57,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T07:16:58,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T07:16:58,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:58,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T07:16:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:58,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/e68a9d2736e44561980e67a164ea9927 is 50, key is test_row_0/A:col10/1731568616968/Put/seqid=0 2024-11-14T07:16:58,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742077_1253 (size=12101) 2024-11-14T07:16:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:58,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568678154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568678154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568678154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568678155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568678160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T07:16:58,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568678261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568678261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568678272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568678272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568678272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/e68a9d2736e44561980e67a164ea9927 2024-11-14T07:16:58,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/6b894426e65e4bfca2cdfb8f4eb87247 is 50, key is test_row_0/B:col10/1731568616968/Put/seqid=0 2024-11-14T07:16:58,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742078_1254 (size=12101) 2024-11-14T07:16:58,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568678472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568678472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568678481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568678482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568678482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T07:16:58,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568678782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568678782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568678792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568678792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568678792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:58,851 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/6b894426e65e4bfca2cdfb8f4eb87247 2024-11-14T07:16:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/a86f648d650d46c19966c61b551a6a06 is 50, key is test_row_0/C:col10/1731568616968/Put/seqid=0 2024-11-14T07:16:58,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742079_1255 (size=12101) 2024-11-14T07:16:58,861 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/a86f648d650d46c19966c61b551a6a06 2024-11-14T07:16:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/e68a9d2736e44561980e67a164ea9927 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/e68a9d2736e44561980e67a164ea9927 2024-11-14T07:16:58,869 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/e68a9d2736e44561980e67a164ea9927, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T07:16:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/6b894426e65e4bfca2cdfb8f4eb87247 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6b894426e65e4bfca2cdfb8f4eb87247 2024-11-14T07:16:58,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6b894426e65e4bfca2cdfb8f4eb87247, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T07:16:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/a86f648d650d46c19966c61b551a6a06 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/a86f648d650d46c19966c61b551a6a06 2024-11-14T07:16:58,878 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/a86f648d650d46c19966c61b551a6a06, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T07:16:58,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b5566aea29a31d45dee688596a38fef4 in 863ms, sequenceid=134, compaction requested=true 2024-11-14T07:16:58,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:58,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:58,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T07:16:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T07:16:58,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T07:16:58,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-14T07:16:58,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.0200 sec 2024-11-14T07:16:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T07:16:59,002 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:59,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T07:16:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:16:59,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:59,008 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:59,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:16:59,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:16:59,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T07:16:59,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:59,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:59,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:59,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:59,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:59,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:59,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/7a2c4f40b74c46a992614b561055866a is 50, key is test_row_0/A:col10/1731568618153/Put/seqid=0 2024-11-14T07:16:59,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742080_1256 (size=12151) 2024-11-14T07:16:59,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/7a2c4f40b74c46a992614b561055866a 2024-11-14T07:16:59,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/356be9550aba41dcb8b39f0aa57c946b is 50, key is test_row_0/B:col10/1731568618153/Put/seqid=0 2024-11-14T07:16:59,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742081_1257 (size=12151) 2024-11-14T07:16:59,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/356be9550aba41dcb8b39f0aa57c946b 2024-11-14T07:16:59,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/d35c637d7d3a4349abe9a4a992ddbdff is 50, key is test_row_0/C:col10/1731568618153/Put/seqid=0 2024-11-14T07:16:59,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742082_1258 (size=12151) 2024-11-14T07:16:59,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/d35c637d7d3a4349abe9a4a992ddbdff 2024-11-14T07:16:59,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/7a2c4f40b74c46a992614b561055866a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/7a2c4f40b74c46a992614b561055866a 2024-11-14T07:16:59,225 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/7a2c4f40b74c46a992614b561055866a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T07:16:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/356be9550aba41dcb8b39f0aa57c946b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/356be9550aba41dcb8b39f0aa57c946b 2024-11-14T07:16:59,231 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/356be9550aba41dcb8b39f0aa57c946b, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T07:16:59,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/d35c637d7d3a4349abe9a4a992ddbdff as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/d35c637d7d3a4349abe9a4a992ddbdff 2024-11-14T07:16:59,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/d35c637d7d3a4349abe9a4a992ddbdff, entries=150, sequenceid=151, filesize=11.9 K 2024-11-14T07:16:59,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for b5566aea29a31d45dee688596a38fef4 in 77ms, sequenceid=151, compaction requested=true 2024-11-14T07:16:59,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:16:59,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T07:16:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T07:16:59,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T07:16:59,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 231 msec 2024-11-14T07:16:59,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 237 msec 2024-11-14T07:16:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:16:59,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:16:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:16:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:16:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:16:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:16:59,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/c849fab52fa54efdbc437c9a8944713f is 50, key is test_row_0/A:col10/1731568619305/Put/seqid=0 2024-11-14T07:16:59,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742083_1259 (size=14541) 2024-11-14T07:16:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:16:59,322 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:16:59,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:16:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T07:16:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:16:59,324 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:16:59,325 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:16:59,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:16:59,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/c849fab52fa54efdbc437c9a8944713f 2024-11-14T07:16:59,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/4acf2ddfc7884e96bbcf89af9a85f6f0 is 50, key is test_row_0/B:col10/1731568619305/Put/seqid=0 2024-11-14T07:16:59,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742084_1260 (size=12151) 2024-11-14T07:16:59,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568679326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568679327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9558, totalHeapSize=21976, min timestamp=1731568619318, max timestamp=1731568619337Num uniques 859; 2024-11-14T07:16:59,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731568619318, max timestamp=1731568619337Num uniques 859; 2024-11-14T07:16:59,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731568619318, max timestamp=1731568619337Num uniques 859; 2024-11-14T07:16:59,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39328, new segment data size=9558, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:59,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:59,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:16:59,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568679392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568679396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568679400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:16:59,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568679461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568679471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:16:59,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:59,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568679512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568679512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568679512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:16:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:16:59,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568679671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568679682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568679722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568679722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568679722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/4acf2ddfc7884e96bbcf89af9a85f6f0 2024-11-14T07:16:59,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e7645c1540954b7297baa06c98f7976c is 50, key is test_row_0/C:col10/1731568619305/Put/seqid=0 2024-11-14T07:16:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742085_1261 (size=12151) 2024-11-14T07:16:59,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:16:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:16:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:16:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:16:59,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:16:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:16:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568679981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:16:59,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:16:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568679992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:16:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:16:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568680032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568680032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568680032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:17:00,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:00,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:00,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:00,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:00,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:00,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e7645c1540954b7297baa06c98f7976c 2024-11-14T07:17:00,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/c849fab52fa54efdbc437c9a8944713f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/c849fab52fa54efdbc437c9a8944713f 2024-11-14T07:17:00,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/c849fab52fa54efdbc437c9a8944713f, entries=200, sequenceid=162, filesize=14.2 K 2024-11-14T07:17:00,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/4acf2ddfc7884e96bbcf89af9a85f6f0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4acf2ddfc7884e96bbcf89af9a85f6f0 2024-11-14T07:17:00,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4acf2ddfc7884e96bbcf89af9a85f6f0, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T07:17:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e7645c1540954b7297baa06c98f7976c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e7645c1540954b7297baa06c98f7976c 2024-11-14T07:17:00,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e7645c1540954b7297baa06c98f7976c, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T07:17:00,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b5566aea29a31d45dee688596a38fef4 in 861ms, sequenceid=162, compaction requested=true 2024-11-14T07:17:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:00,166 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:17:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:00,166 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:17:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:00,167 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63001 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:17:00,167 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60611 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:17:00,167 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:17:00,167 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:17:00,167 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:00,167 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:00,168 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6820a7ac0ce94b51b2fdba93ec5ad735, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/fc033a1ecd034c0fb357f52da18fcbf8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6b894426e65e4bfca2cdfb8f4eb87247, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/356be9550aba41dcb8b39f0aa57c946b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4acf2ddfc7884e96bbcf89af9a85f6f0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=59.2 K 2024-11-14T07:17:00,168 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/eaf43deb11454edb9885e314cca2b622, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/0399afccdea34d75b6e55a69b1f35e9c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/e68a9d2736e44561980e67a164ea9927, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/7a2c4f40b74c46a992614b561055866a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/c849fab52fa54efdbc437c9a8944713f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=61.5 K 2024-11-14T07:17:00,168 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaf43deb11454edb9885e314cca2b622, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731568615670 2024-11-14T07:17:00,168 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6820a7ac0ce94b51b2fdba93ec5ad735, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731568615670 2024-11-14T07:17:00,168 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting fc033a1ecd034c0fb357f52da18fcbf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731568616320 2024-11-14T07:17:00,168 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0399afccdea34d75b6e55a69b1f35e9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731568616320 2024-11-14T07:17:00,169 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting e68a9d2736e44561980e67a164ea9927, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731568616965 2024-11-14T07:17:00,169 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b894426e65e4bfca2cdfb8f4eb87247, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731568616965 2024-11-14T07:17:00,169 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a2c4f40b74c46a992614b561055866a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731568618153 2024-11-14T07:17:00,169 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 356be9550aba41dcb8b39f0aa57c946b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731568618153 2024-11-14T07:17:00,170 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 4acf2ddfc7884e96bbcf89af9a85f6f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731568619301 2024-11-14T07:17:00,170 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting c849fab52fa54efdbc437c9a8944713f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731568619298 2024-11-14T07:17:00,179 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:00,179 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/fcd643104e43443fab04be2baecd2129 is 50, key is test_row_0/A:col10/1731568619305/Put/seqid=0 2024-11-14T07:17:00,183 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:00,184 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/4856389d20ac45908c957288ddad816d is 50, key is test_row_0/B:col10/1731568619305/Put/seqid=0 2024-11-14T07:17:00,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742086_1262 (size=12527) 2024-11-14T07:17:00,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742087_1263 (size=12527) 2024-11-14T07:17:00,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:17:00,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:00,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T07:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/af96222db37d47ad98e6fccb57a01250 is 50, key is test_row_0/A:col10/1731568619368/Put/seqid=0 2024-11-14T07:17:00,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742088_1264 (size=12151) 2024-11-14T07:17:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:17:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:00,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568680525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568680526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568680561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568680561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568680562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,594 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/fcd643104e43443fab04be2baecd2129 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/fcd643104e43443fab04be2baecd2129 2024-11-14T07:17:00,594 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/4856389d20ac45908c957288ddad816d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4856389d20ac45908c957288ddad816d 2024-11-14T07:17:00,599 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into 4856389d20ac45908c957288ddad816d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:00,599 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into fcd643104e43443fab04be2baecd2129(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:00,599 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:00,599 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:00,599 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=11, startTime=1731568620166; duration=0sec 2024-11-14T07:17:00,599 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=11, startTime=1731568620166; duration=0sec 2024-11-14T07:17:00,599 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:00,599 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:00,599 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:17:00,599 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:17:00,599 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:17:00,601 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60611 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:17:00,601 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:17:00,601 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:00,601 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/00c8b614b3b44e58b80dee81579e7c72, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e271cf630e2441b3b71aa4f7d249ec2e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/a86f648d650d46c19966c61b551a6a06, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/d35c637d7d3a4349abe9a4a992ddbdff, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e7645c1540954b7297baa06c98f7976c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=59.2 K 2024-11-14T07:17:00,601 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 00c8b614b3b44e58b80dee81579e7c72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731568615670 2024-11-14T07:17:00,602 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e271cf630e2441b3b71aa4f7d249ec2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731568616320 2024-11-14T07:17:00,602 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a86f648d650d46c19966c61b551a6a06, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731568616965 2024-11-14T07:17:00,602 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d35c637d7d3a4349abe9a4a992ddbdff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731568618153 2024-11-14T07:17:00,603 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e7645c1540954b7297baa06c98f7976c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731568619301 2024-11-14T07:17:00,613 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:00,614 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/1099b86c50d844a499bdac3c87b3c5c8 is 50, key is test_row_0/C:col10/1731568619305/Put/seqid=0 2024-11-14T07:17:00,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742089_1265 (size=12527) 2024-11-14T07:17:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568680641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568680641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,651 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/af96222db37d47ad98e6fccb57a01250 2024-11-14T07:17:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/b28e6290606a4041890ede9e9af5dd64 is 50, key is test_row_0/B:col10/1731568619368/Put/seqid=0 2024-11-14T07:17:00,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742090_1266 (size=12151) 2024-11-14T07:17:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568680852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:00,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568680852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,023 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/1099b86c50d844a499bdac3c87b3c5c8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/1099b86c50d844a499bdac3c87b3c5c8 2024-11-14T07:17:01,027 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into 1099b86c50d844a499bdac3c87b3c5c8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:01,027 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:01,027 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=11, startTime=1731568620166; duration=0sec 2024-11-14T07:17:01,027 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:01,027 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:17:01,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/b28e6290606a4041890ede9e9af5dd64 2024-11-14T07:17:01,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/01370b176ecc4a989a2b46f8ad3645f7 is 50, key is test_row_0/C:col10/1731568619368/Put/seqid=0 2024-11-14T07:17:01,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742091_1267 (size=12151) 2024-11-14T07:17:01,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568681162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568681162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/01370b176ecc4a989a2b46f8ad3645f7 2024-11-14T07:17:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:17:01,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/af96222db37d47ad98e6fccb57a01250 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/af96222db37d47ad98e6fccb57a01250 2024-11-14T07:17:01,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/af96222db37d47ad98e6fccb57a01250, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T07:17:01,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/b28e6290606a4041890ede9e9af5dd64 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/b28e6290606a4041890ede9e9af5dd64 2024-11-14T07:17:01,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/b28e6290606a4041890ede9e9af5dd64, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T07:17:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/01370b176ecc4a989a2b46f8ad3645f7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/01370b176ecc4a989a2b46f8ad3645f7 2024-11-14T07:17:01,488 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/01370b176ecc4a989a2b46f8ad3645f7, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T07:17:01,489 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for b5566aea29a31d45dee688596a38fef4 in 1248ms, sequenceid=197, compaction requested=false 2024-11-14T07:17:01,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:01,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:01,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T07:17:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T07:17:01,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T07:17:01,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1650 sec 2024-11-14T07:17:01,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1710 sec 2024-11-14T07:17:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:01,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:17:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:01,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/152350dc5a614463ae1d1044282cd91a is 50, key is test_row_0/A:col10/1731568620524/Put/seqid=0 2024-11-14T07:17:01,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742092_1268 (size=9757) 2024-11-14T07:17:01,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568681602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568681603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568681605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568681672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568681672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568681714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568681714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568681714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568681921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568681921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568681921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:01,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/152350dc5a614463ae1d1044282cd91a 2024-11-14T07:17:01,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5ab786604f274aa68013c6fc7ac2771f is 50, key is test_row_0/B:col10/1731568620524/Put/seqid=0 2024-11-14T07:17:01,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742093_1269 (size=9757) 2024-11-14T07:17:01,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5ab786604f274aa68013c6fc7ac2771f 2024-11-14T07:17:01,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/aa5ec563c6aa4552b8b3351478bb63f4 is 50, key is test_row_0/C:col10/1731568620524/Put/seqid=0 2024-11-14T07:17:02,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742094_1270 (size=9757) 2024-11-14T07:17:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568682232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568682232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568682232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/aa5ec563c6aa4552b8b3351478bb63f4 2024-11-14T07:17:02,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/152350dc5a614463ae1d1044282cd91a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/152350dc5a614463ae1d1044282cd91a 2024-11-14T07:17:02,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/152350dc5a614463ae1d1044282cd91a, entries=100, sequenceid=212, filesize=9.5 K 2024-11-14T07:17:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5ab786604f274aa68013c6fc7ac2771f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5ab786604f274aa68013c6fc7ac2771f 2024-11-14T07:17:02,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5ab786604f274aa68013c6fc7ac2771f, entries=100, sequenceid=212, filesize=9.5 K 2024-11-14T07:17:02,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/aa5ec563c6aa4552b8b3351478bb63f4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/aa5ec563c6aa4552b8b3351478bb63f4 2024-11-14T07:17:02,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/aa5ec563c6aa4552b8b3351478bb63f4, entries=100, sequenceid=212, filesize=9.5 K 2024-11-14T07:17:02,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b5566aea29a31d45dee688596a38fef4 in 844ms, sequenceid=212, compaction requested=true 2024-11-14T07:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:02,418 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:02,418 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:02,420 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:02,420 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:02,420 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:17:02,420 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:17:02,420 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:02,420 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:02,421 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4856389d20ac45908c957288ddad816d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/b28e6290606a4041890ede9e9af5dd64, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5ab786604f274aa68013c6fc7ac2771f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=33.6 K 2024-11-14T07:17:02,421 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/fcd643104e43443fab04be2baecd2129, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/af96222db37d47ad98e6fccb57a01250, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/152350dc5a614463ae1d1044282cd91a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=33.6 K 2024-11-14T07:17:02,421 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 4856389d20ac45908c957288ddad816d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731568619301 2024-11-14T07:17:02,421 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcd643104e43443fab04be2baecd2129, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731568619301 2024-11-14T07:17:02,421 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting b28e6290606a4041890ede9e9af5dd64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731568619368 2024-11-14T07:17:02,421 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting af96222db37d47ad98e6fccb57a01250, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731568619368 2024-11-14T07:17:02,421 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab786604f274aa68013c6fc7ac2771f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731568620524 2024-11-14T07:17:02,421 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152350dc5a614463ae1d1044282cd91a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731568620524 2024-11-14T07:17:02,427 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:02,428 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/d11e40fd5e354898b27d99118f14d660 is 50, key is test_row_0/A:col10/1731568620524/Put/seqid=0 2024-11-14T07:17:02,430 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:02,430 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5201f8a79f674f6d926d52818e6348e0 is 50, key is test_row_0/B:col10/1731568620524/Put/seqid=0 2024-11-14T07:17:02,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742095_1271 (size=12629) 2024-11-14T07:17:02,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742096_1272 (size=12629) 2024-11-14T07:17:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:02,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T07:17:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:02,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/6f4efc9361ed4980b3cb022a2267f640 is 50, key is test_row_0/A:col10/1731568621604/Put/seqid=0 2024-11-14T07:17:02,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742097_1273 (size=12151) 2024-11-14T07:17:02,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568682703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568682705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568682742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568682742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568682751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568682812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568682812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:02,841 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5201f8a79f674f6d926d52818e6348e0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5201f8a79f674f6d926d52818e6348e0 2024-11-14T07:17:02,841 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/d11e40fd5e354898b27d99118f14d660 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/d11e40fd5e354898b27d99118f14d660 2024-11-14T07:17:02,849 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into 5201f8a79f674f6d926d52818e6348e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:02,849 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into d11e40fd5e354898b27d99118f14d660(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:02,849 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:02,849 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:02,849 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=13, startTime=1731568622418; duration=0sec 2024-11-14T07:17:02,849 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=13, startTime=1731568622418; duration=0sec 2024-11-14T07:17:02,849 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:02,849 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:17:02,849 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:02,849 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:17:02,849 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:02,850 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:02,850 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:17:02,850 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:02,850 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/1099b86c50d844a499bdac3c87b3c5c8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/01370b176ecc4a989a2b46f8ad3645f7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/aa5ec563c6aa4552b8b3351478bb63f4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=33.6 K 2024-11-14T07:17:02,850 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 1099b86c50d844a499bdac3c87b3c5c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731568619301 2024-11-14T07:17:02,851 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 01370b176ecc4a989a2b46f8ad3645f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731568619368 2024-11-14T07:17:02,851 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting aa5ec563c6aa4552b8b3351478bb63f4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731568620524 2024-11-14T07:17:02,859 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:02,859 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/67fe63afe4da447b9cdbc2d45fc4111f is 50, key is test_row_0/C:col10/1731568620524/Put/seqid=0 2024-11-14T07:17:02,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742098_1274 (size=12629) 2024-11-14T07:17:03,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568683023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568683023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/6f4efc9361ed4980b3cb022a2267f640 2024-11-14T07:17:03,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5a5914bd06164003a4715dc7427048f4 is 50, key is test_row_0/B:col10/1731568621604/Put/seqid=0 2024-11-14T07:17:03,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742099_1275 (size=12151) 2024-11-14T07:17:03,269 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/67fe63afe4da447b9cdbc2d45fc4111f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/67fe63afe4da447b9cdbc2d45fc4111f 2024-11-14T07:17:03,273 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into 67fe63afe4da447b9cdbc2d45fc4111f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:03,273 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:03,273 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=13, startTime=1731568622418; duration=0sec 2024-11-14T07:17:03,273 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:03,273 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:17:03,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568683331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568683331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:17:03,492 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:03,492 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:03,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5a5914bd06164003a4715dc7427048f4 2024-11-14T07:17:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T07:17:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:17:03,525 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:03,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:03,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:03,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/5411bf7dd82e45b899d6d5979dbdb13f is 50, key is test_row_0/C:col10/1731568621604/Put/seqid=0 2024-11-14T07:17:03,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742100_1276 (size=12151) 2024-11-14T07:17:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:17:03,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T07:17:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:03,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568683752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,755 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568683762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568683762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,764 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,765 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T07:17:03,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:03,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:03,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:03,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:03,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:17:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568683842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568683842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:03,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/5411bf7dd82e45b899d6d5979dbdb13f 2024-11-14T07:17:03,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/6f4efc9361ed4980b3cb022a2267f640 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/6f4efc9361ed4980b3cb022a2267f640 2024-11-14T07:17:03,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/6f4efc9361ed4980b3cb022a2267f640, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T07:17:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5a5914bd06164003a4715dc7427048f4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5a5914bd06164003a4715dc7427048f4 2024-11-14T07:17:03,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5a5914bd06164003a4715dc7427048f4, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T07:17:03,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/5411bf7dd82e45b899d6d5979dbdb13f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5411bf7dd82e45b899d6d5979dbdb13f 2024-11-14T07:17:03,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5411bf7dd82e45b899d6d5979dbdb13f, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T07:17:03,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b5566aea29a31d45dee688596a38fef4 in 1271ms, sequenceid=233, compaction requested=false 2024-11-14T07:17:03,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:03,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T07:17:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:03,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:17:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:03,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/20056a3755434e7b8fcbbeac0ce75adb is 50, key is test_row_0/A:col10/1731568622687/Put/seqid=0 2024-11-14T07:17:04,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742101_1277 (size=12151) 2024-11-14T07:17:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:17:04,402 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/20056a3755434e7b8fcbbeac0ce75adb 2024-11-14T07:17:04,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/0e6425c7fe1d411fbd28dde980647c7c is 50, key is test_row_0/B:col10/1731568622687/Put/seqid=0 2024-11-14T07:17:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742102_1278 (size=12151) 2024-11-14T07:17:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:17:04,816 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/0e6425c7fe1d411fbd28dde980647c7c 2024-11-14T07:17:04,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/31b29ae73aeb420ea7a0a65346b8ff34 is 50, key is test_row_0/C:col10/1731568622687/Put/seqid=0 2024-11-14T07:17:04,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742103_1279 (size=12151) 2024-11-14T07:17:04,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:04,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:04,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731568624881, max timestamp=1731568624913Num uniques 859; 2024-11-14T07:17:04,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731568624881, max timestamp=1731568624913Num uniques 859; 2024-11-14T07:17:04,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:04,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:04,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568624881, max timestamp=1731568624913Num uniques 859; 2024-11-14T07:17:04,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:17:04,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568684963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:04,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568684965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568685081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568685081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,227 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/31b29ae73aeb420ea7a0a65346b8ff34 2024-11-14T07:17:05,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/20056a3755434e7b8fcbbeac0ce75adb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/20056a3755434e7b8fcbbeac0ce75adb 2024-11-14T07:17:05,235 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/20056a3755434e7b8fcbbeac0ce75adb, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T07:17:05,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/0e6425c7fe1d411fbd28dde980647c7c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/0e6425c7fe1d411fbd28dde980647c7c 2024-11-14T07:17:05,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/0e6425c7fe1d411fbd28dde980647c7c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T07:17:05,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/31b29ae73aeb420ea7a0a65346b8ff34 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/31b29ae73aeb420ea7a0a65346b8ff34 2024-11-14T07:17:05,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/31b29ae73aeb420ea7a0a65346b8ff34, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T07:17:05,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for b5566aea29a31d45dee688596a38fef4 in 1259ms, sequenceid=247, compaction requested=true 2024-11-14T07:17:05,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:05,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:05,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T07:17:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T07:17:05,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T07:17:05,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-11-14T07:17:05,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7560 sec 2024-11-14T07:17:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:05,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-14T07:17:05,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:05,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:05,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:05,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:05,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/34b767f883244e5b83bb566a4356958c is 50, key is test_row_0/A:col10/1731568625294/Put/seqid=0 2024-11-14T07:17:05,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742104_1280 (size=14741) 2024-11-14T07:17:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568685308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568685312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568685422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568685422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568685632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568685632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:17:05,682 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:05,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T07:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:17:05,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:05,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:05,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:05,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/34b767f883244e5b83bb566a4356958c 2024-11-14T07:17:05,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/339875a0a7b14dd1bf2a62de086bbf8a is 50, key is test_row_0/B:col10/1731568625294/Put/seqid=0 2024-11-14T07:17:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742105_1281 (size=12301) 2024-11-14T07:17:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568685771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,775 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568685782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,786 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:17:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568685792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,795 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:17:05,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:05,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:05,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568685942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568685942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:05,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:17:05,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:05,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:05,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:05,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:05,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:17:06,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/339875a0a7b14dd1bf2a62de086bbf8a 2024-11-14T07:17:06,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2bbad79ec3b14e4280c5c10a2b393751 is 50, key is test_row_0/C:col10/1731568625294/Put/seqid=0 2024-11-14T07:17:06,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742106_1282 (size=12301) 2024-11-14T07:17:06,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:17:06,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:06,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:17:06,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:06,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:17:06,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:17:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:06,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568686452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:06,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568686452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:06,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2bbad79ec3b14e4280c5c10a2b393751 2024-11-14T07:17:06,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/34b767f883244e5b83bb566a4356958c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/34b767f883244e5b83bb566a4356958c 2024-11-14T07:17:06,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/34b767f883244e5b83bb566a4356958c, entries=200, sequenceid=285, filesize=14.4 K 2024-11-14T07:17:06,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/339875a0a7b14dd1bf2a62de086bbf8a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/339875a0a7b14dd1bf2a62de086bbf8a 2024-11-14T07:17:06,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/339875a0a7b14dd1bf2a62de086bbf8a, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T07:17:06,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2bbad79ec3b14e4280c5c10a2b393751 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2bbad79ec3b14e4280c5c10a2b393751 2024-11-14T07:17:06,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2bbad79ec3b14e4280c5c10a2b393751, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T07:17:06,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~415.72 KB/425696, currentSize=33.54 KB/34350 for b5566aea29a31d45dee688596a38fef4 in 1258ms, sequenceid=285, compaction requested=true 2024-11-14T07:17:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:06,553 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:17:06,553 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:17:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:06,554 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:17:06,554 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:17:06,554 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,554 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5201f8a79f674f6d926d52818e6348e0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5a5914bd06164003a4715dc7427048f4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/0e6425c7fe1d411fbd28dde980647c7c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/339875a0a7b14dd1bf2a62de086bbf8a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=48.1 K 2024-11-14T07:17:06,555 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51672 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:17:06,555 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:17:06,555 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,555 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/d11e40fd5e354898b27d99118f14d660, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/6f4efc9361ed4980b3cb022a2267f640, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/20056a3755434e7b8fcbbeac0ce75adb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/34b767f883244e5b83bb566a4356958c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=50.5 K 2024-11-14T07:17:06,555 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5201f8a79f674f6d926d52818e6348e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731568619393 2024-11-14T07:17:06,555 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting d11e40fd5e354898b27d99118f14d660, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731568619393 2024-11-14T07:17:06,555 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a5914bd06164003a4715dc7427048f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731568621598 2024-11-14T07:17:06,556 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f4efc9361ed4980b3cb022a2267f640, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731568621598 2024-11-14T07:17:06,556 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e6425c7fe1d411fbd28dde980647c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568622687 2024-11-14T07:17:06,556 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20056a3755434e7b8fcbbeac0ce75adb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568622687 2024-11-14T07:17:06,556 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 339875a0a7b14dd1bf2a62de086bbf8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731568624962 2024-11-14T07:17:06,556 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b767f883244e5b83bb566a4356958c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731568624951 2024-11-14T07:17:06,565 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:06,565 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/65ba738c64974f61a1042ad2aeedc972 is 50, key is test_row_0/B:col10/1731568625294/Put/seqid=0 2024-11-14T07:17:06,571 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:06,571 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/29e6743df68b42c9a1aecc5b257b4646 is 50, key is test_row_0/A:col10/1731568625294/Put/seqid=0 2024-11-14T07:17:06,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742107_1283 (size=12915) 2024-11-14T07:17:06,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742108_1284 (size=12915) 2024-11-14T07:17:06,591 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/65ba738c64974f61a1042ad2aeedc972 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/65ba738c64974f61a1042ad2aeedc972 2024-11-14T07:17:06,596 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into 65ba738c64974f61a1042ad2aeedc972(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:06,596 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:06,596 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=12, startTime=1731568626553; duration=0sec 2024-11-14T07:17:06,596 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:06,596 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:17:06,597 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:17:06,598 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:17:06,598 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:17:06,598 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,598 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/67fe63afe4da447b9cdbc2d45fc4111f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5411bf7dd82e45b899d6d5979dbdb13f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/31b29ae73aeb420ea7a0a65346b8ff34, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2bbad79ec3b14e4280c5c10a2b393751] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=48.1 K 2024-11-14T07:17:06,599 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 67fe63afe4da447b9cdbc2d45fc4111f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731568619393 2024-11-14T07:17:06,600 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5411bf7dd82e45b899d6d5979dbdb13f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731568621598 2024-11-14T07:17:06,601 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 31b29ae73aeb420ea7a0a65346b8ff34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568622687 2024-11-14T07:17:06,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:06,601 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T07:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:06,602 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbad79ec3b14e4280c5c10a2b393751, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731568624962 2024-11-14T07:17:06,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/28437756d19547e4b69f110ad401ffc7 is 50, key is test_row_0/A:col10/1731568625311/Put/seqid=0 2024-11-14T07:17:06,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742109_1285 (size=12301) 2024-11-14T07:17:06,618 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:06,619 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2d3e0dc33f9246c5a53b6b0da32f9fe2 is 50, key is test_row_0/C:col10/1731568625294/Put/seqid=0 2024-11-14T07:17:06,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742110_1286 (size=12915) 2024-11-14T07:17:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:17:06,992 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/29e6743df68b42c9a1aecc5b257b4646 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/29e6743df68b42c9a1aecc5b257b4646 2024-11-14T07:17:07,011 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into 29e6743df68b42c9a1aecc5b257b4646(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:07,011 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:07,011 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=12, startTime=1731568626553; duration=0sec 2024-11-14T07:17:07,011 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:07,011 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:17:07,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/28437756d19547e4b69f110ad401ffc7 2024-11-14T07:17:07,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/36374e2752ba4e65b2da265f473e2704 is 50, key is test_row_0/B:col10/1731568625311/Put/seqid=0 2024-11-14T07:17:07,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742111_1287 (size=12301) 2024-11-14T07:17:07,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/36374e2752ba4e65b2da265f473e2704 2024-11-14T07:17:07,029 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2d3e0dc33f9246c5a53b6b0da32f9fe2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2d3e0dc33f9246c5a53b6b0da32f9fe2 2024-11-14T07:17:07,033 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into 2d3e0dc33f9246c5a53b6b0da32f9fe2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:07,033 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:07,033 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=12, startTime=1731568626553; duration=0sec 2024-11-14T07:17:07,033 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:07,034 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:17:07,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2a6be27867dd43ada00bb15765779532 is 50, key is test_row_0/C:col10/1731568625311/Put/seqid=0 2024-11-14T07:17:07,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742112_1288 (size=12301) 2024-11-14T07:17:07,438 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2a6be27867dd43ada00bb15765779532 2024-11-14T07:17:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/28437756d19547e4b69f110ad401ffc7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/28437756d19547e4b69f110ad401ffc7 2024-11-14T07:17:07,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/28437756d19547e4b69f110ad401ffc7, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T07:17:07,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/36374e2752ba4e65b2da265f473e2704 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/36374e2752ba4e65b2da265f473e2704 2024-11-14T07:17:07,450 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/36374e2752ba4e65b2da265f473e2704, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T07:17:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/2a6be27867dd43ada00bb15765779532 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2a6be27867dd43ada00bb15765779532 2024-11-14T07:17:07,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2a6be27867dd43ada00bb15765779532, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T07:17:07,456 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b5566aea29a31d45dee688596a38fef4 in 855ms, sequenceid=294, compaction requested=false 2024-11-14T07:17:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T07:17:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T07:17:07,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T07:17:07,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-14T07:17:07,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7780 sec 2024-11-14T07:17:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:07,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:17:07,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:07,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:07,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:07,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:07,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:07,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:07,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/1bacd38ce4454f1da4761208af8d2ba2 is 50, key is test_row_0/A:col10/1731568627462/Put/seqid=0 2024-11-14T07:17:07,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742113_1289 (size=17181) 2024-11-14T07:17:07,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/1bacd38ce4454f1da4761208af8d2ba2 2024-11-14T07:17:07,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/9b120132b383416981ecbd8623368ade is 50, key is test_row_0/B:col10/1731568627462/Put/seqid=0 2024-11-14T07:17:07,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568627504, max timestamp=1731568627522Num uniques 859; 2024-11-14T07:17:07,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:17:07,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568627504, max timestamp=1731568627522Num uniques 859; 2024-11-14T07:17:07,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568627504, max timestamp=1731568627522Num uniques 859; 2024-11-14T07:17:07,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:17:07,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:17:07,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742114_1290 (size=12301) 2024-11-14T07:17:07,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/9b120132b383416981ecbd8623368ade 2024-11-14T07:17:07,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/ee9f96b813ed42d9a7b636e85905186d is 50, key is test_row_0/C:col10/1731568627462/Put/seqid=0 2024-11-14T07:17:07,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742115_1291 (size=12301) 2024-11-14T07:17:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568687563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568687566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:07,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568687672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:07,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568687682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:17:07,832 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:07,833 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-14T07:17:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:17:07,834 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:07,835 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:07,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:07,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568687882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568687892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:17:07,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/ee9f96b813ed42d9a7b636e85905186d 2024-11-14T07:17:07,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/1bacd38ce4454f1da4761208af8d2ba2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/1bacd38ce4454f1da4761208af8d2ba2 2024-11-14T07:17:07,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/1bacd38ce4454f1da4761208af8d2ba2, entries=250, sequenceid=307, filesize=16.8 K 2024-11-14T07:17:07,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/9b120132b383416981ecbd8623368ade as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/9b120132b383416981ecbd8623368ade 2024-11-14T07:17:07,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/9b120132b383416981ecbd8623368ade, entries=150, sequenceid=307, filesize=12.0 K 2024-11-14T07:17:07,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/ee9f96b813ed42d9a7b636e85905186d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/ee9f96b813ed42d9a7b636e85905186d 2024-11-14T07:17:07,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/ee9f96b813ed42d9a7b636e85905186d, entries=150, sequenceid=307, filesize=12.0 K 2024-11-14T07:17:07,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b5566aea29a31d45dee688596a38fef4 in 505ms, sequenceid=307, compaction requested=true 2024-11-14T07:17:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:07,976 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:07,976 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:07,977 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:07,977 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:07,977 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:17:07,977 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:17:07,977 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:07,977 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:07,977 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/29e6743df68b42c9a1aecc5b257b4646, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/28437756d19547e4b69f110ad401ffc7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/1bacd38ce4454f1da4761208af8d2ba2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=41.4 K 2024-11-14T07:17:07,977 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/65ba738c64974f61a1042ad2aeedc972, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/36374e2752ba4e65b2da265f473e2704, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/9b120132b383416981ecbd8623368ade] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=36.6 K 2024-11-14T07:17:07,977 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ba738c64974f61a1042ad2aeedc972, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731568624962 2024-11-14T07:17:07,977 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e6743df68b42c9a1aecc5b257b4646, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731568624962 2024-11-14T07:17:07,978 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 36374e2752ba4e65b2da265f473e2704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731568625305 2024-11-14T07:17:07,978 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28437756d19547e4b69f110ad401ffc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731568625305 2024-11-14T07:17:07,978 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b120132b383416981ecbd8623368ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731568627462 2024-11-14T07:17:07,978 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bacd38ce4454f1da4761208af8d2ba2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731568627462 2024-11-14T07:17:07,984 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:07,984 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:07,985 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/de6957d37f5148baba30ac6a04ec3a5e is 50, key is test_row_0/B:col10/1731568627462/Put/seqid=0 2024-11-14T07:17:07,985 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/a3665e01e7be4c8693605a18b7fdb9e7 is 50, key is test_row_0/A:col10/1731568627462/Put/seqid=0 2024-11-14T07:17:07,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T07:17:07,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:07,987 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-14T07:17:07,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:08,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742116_1292 (size=13017) 2024-11-14T07:17:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/9a6bd5a739d94614a9ba88944d2ae26e is 50, key is test_row_0/A:col10/1731568627561/Put/seqid=0 2024-11-14T07:17:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742117_1293 (size=13017) 2024-11-14T07:17:08,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742118_1294 (size=12301) 2024-11-14T07:17:08,007 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/9a6bd5a739d94614a9ba88944d2ae26e 2024-11-14T07:17:08,010 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/a3665e01e7be4c8693605a18b7fdb9e7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/a3665e01e7be4c8693605a18b7fdb9e7 2024-11-14T07:17:08,010 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/de6957d37f5148baba30ac6a04ec3a5e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/de6957d37f5148baba30ac6a04ec3a5e 2024-11-14T07:17:08,017 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into de6957d37f5148baba30ac6a04ec3a5e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:08,017 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:08,017 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=13, startTime=1731568627976; duration=0sec 2024-11-14T07:17:08,017 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:08,017 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:17:08,017 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:08,018 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into a3665e01e7be4c8693605a18b7fdb9e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:08,018 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:08,018 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=13, startTime=1731568627976; duration=0sec 2024-11-14T07:17:08,018 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:08,018 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:17:08,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/cd45ad9131dd4f9fb8c00f5a570db41d is 50, key is test_row_0/B:col10/1731568627561/Put/seqid=0 2024-11-14T07:17:08,019 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:08,019 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:17:08,020 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:08,020 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2d3e0dc33f9246c5a53b6b0da32f9fe2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2a6be27867dd43ada00bb15765779532, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/ee9f96b813ed42d9a7b636e85905186d] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=36.6 K 2024-11-14T07:17:08,020 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3e0dc33f9246c5a53b6b0da32f9fe2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731568624962 2024-11-14T07:17:08,020 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6be27867dd43ada00bb15765779532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731568625305 2024-11-14T07:17:08,021 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ee9f96b813ed42d9a7b636e85905186d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731568627462 2024-11-14T07:17:08,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742119_1295 (size=12301) 2024-11-14T07:17:08,027 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:08,027 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/b4b92712867e4d47b5d20388df2aac08 is 50, key is test_row_0/C:col10/1731568627462/Put/seqid=0 2024-11-14T07:17:08,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742120_1296 (size=13017) 2024-11-14T07:17:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:17:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:08,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:08,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568688216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568688217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568688322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568688332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/cd45ad9131dd4f9fb8c00f5a570db41d 2024-11-14T07:17:08,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/9aa1f2828b08423dbda22ebe1093d1d6 is 50, key is test_row_0/C:col10/1731568627561/Put/seqid=0 2024-11-14T07:17:08,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742121_1297 (size=12301) 2024-11-14T07:17:08,439 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/9aa1f2828b08423dbda22ebe1093d1d6 2024-11-14T07:17:08,442 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/b4b92712867e4d47b5d20388df2aac08 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/b4b92712867e4d47b5d20388df2aac08 2024-11-14T07:17:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/9a6bd5a739d94614a9ba88944d2ae26e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/9a6bd5a739d94614a9ba88944d2ae26e 2024-11-14T07:17:08,448 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into b4b92712867e4d47b5d20388df2aac08(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:08,448 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:08,448 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=13, startTime=1731568627976; duration=0sec 2024-11-14T07:17:08,448 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/9a6bd5a739d94614a9ba88944d2ae26e, entries=150, sequenceid=343, filesize=12.0 K 2024-11-14T07:17:08,448 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:08,448 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:17:08,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/cd45ad9131dd4f9fb8c00f5a570db41d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/cd45ad9131dd4f9fb8c00f5a570db41d 2024-11-14T07:17:08,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/cd45ad9131dd4f9fb8c00f5a570db41d, entries=150, sequenceid=343, filesize=12.0 K 2024-11-14T07:17:08,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/9aa1f2828b08423dbda22ebe1093d1d6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9aa1f2828b08423dbda22ebe1093d1d6 2024-11-14T07:17:08,456 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9aa1f2828b08423dbda22ebe1093d1d6, entries=150, sequenceid=343, filesize=12.0 K 2024-11-14T07:17:08,458 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for b5566aea29a31d45dee688596a38fef4 in 471ms, sequenceid=343, compaction requested=false 2024-11-14T07:17:08,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:08,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:08,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-14T07:17:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-14T07:17:08,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T07:17:08,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 624 msec 2024-11-14T07:17:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:17:08,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 628 msec 2024-11-14T07:17:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:08,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:17:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:08,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f07860c03b164e5d8e0ca27eb2ebacce is 50, key is test_row_0/A:col10/1731568628216/Put/seqid=0 2024-11-14T07:17:08,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742122_1298 (size=14741) 2024-11-14T07:17:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568688582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568688582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568688692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568688692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568688902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568688902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:08,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f07860c03b164e5d8e0ca27eb2ebacce 2024-11-14T07:17:08,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/2fa7f4364bde41cc8249a6e9f0fd2638 is 50, key is test_row_0/B:col10/1731568628216/Put/seqid=0 2024-11-14T07:17:08,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742123_1299 (size=12301) 2024-11-14T07:17:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:17:08,972 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:08,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-14T07:17:08,973 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:17:08,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:08,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:17:09,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T07:17:09,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:09,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:09,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568689212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568689212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T07:17:09,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:09,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:09,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:17:09,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/2fa7f4364bde41cc8249a6e9f0fd2638 2024-11-14T07:17:09,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/824b80ddb26347839cc8d9af26e8772e is 50, key is test_row_0/C:col10/1731568628216/Put/seqid=0 2024-11-14T07:17:09,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742124_1300 (size=12301) 2024-11-14T07:17:09,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/824b80ddb26347839cc8d9af26e8772e 2024-11-14T07:17:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/f07860c03b164e5d8e0ca27eb2ebacce as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07860c03b164e5d8e0ca27eb2ebacce 2024-11-14T07:17:09,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07860c03b164e5d8e0ca27eb2ebacce, entries=200, sequenceid=358, filesize=14.4 K 2024-11-14T07:17:09,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/2fa7f4364bde41cc8249a6e9f0fd2638 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/2fa7f4364bde41cc8249a6e9f0fd2638 2024-11-14T07:17:09,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/2fa7f4364bde41cc8249a6e9f0fd2638, entries=150, sequenceid=358, filesize=12.0 K 2024-11-14T07:17:09,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/824b80ddb26347839cc8d9af26e8772e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/824b80ddb26347839cc8d9af26e8772e 2024-11-14T07:17:09,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/824b80ddb26347839cc8d9af26e8772e, entries=150, sequenceid=358, filesize=12.0 K 2024-11-14T07:17:09,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b5566aea29a31d45dee688596a38fef4 in 871ms, sequenceid=358, compaction requested=true 2024-11-14T07:17:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:09,405 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:09,405 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:09,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:09,407 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:09,407 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:17:09,407 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,408 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/de6957d37f5148baba30ac6a04ec3a5e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/cd45ad9131dd4f9fb8c00f5a570db41d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/2fa7f4364bde41cc8249a6e9f0fd2638] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=36.7 K 2024-11-14T07:17:09,408 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:09,408 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:17:09,408 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,408 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/a3665e01e7be4c8693605a18b7fdb9e7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/9a6bd5a739d94614a9ba88944d2ae26e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07860c03b164e5d8e0ca27eb2ebacce] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=39.1 K 2024-11-14T07:17:09,409 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3665e01e7be4c8693605a18b7fdb9e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731568627462 2024-11-14T07:17:09,409 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting de6957d37f5148baba30ac6a04ec3a5e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731568627462 2024-11-14T07:17:09,410 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a6bd5a739d94614a9ba88944d2ae26e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731568627556 2024-11-14T07:17:09,410 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cd45ad9131dd4f9fb8c00f5a570db41d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731568627556 2024-11-14T07:17:09,410 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f07860c03b164e5d8e0ca27eb2ebacce, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731568628195 2024-11-14T07:17:09,410 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa7f4364bde41cc8249a6e9f0fd2638, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731568628195 2024-11-14T07:17:09,418 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:09,418 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:09,419 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/dc365bae15fc40239c603aeac5127621 is 50, key is test_row_0/A:col10/1731568628216/Put/seqid=0 2024-11-14T07:17:09,419 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5efde1afc1b34da78e8c9194f53621a6 is 50, key is test_row_0/B:col10/1731568628216/Put/seqid=0 2024-11-14T07:17:09,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742125_1301 (size=13119) 2024-11-14T07:17:09,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T07:17:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:17:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:09,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742126_1302 (size=13119) 2024-11-14T07:17:09,446 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/5efde1afc1b34da78e8c9194f53621a6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5efde1afc1b34da78e8c9194f53621a6 2024-11-14T07:17:09,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/92ff4dbafe48487496ea0344a441acc9 is 50, key is test_row_0/A:col10/1731568628581/Put/seqid=0 2024-11-14T07:17:09,453 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into 5efde1afc1b34da78e8c9194f53621a6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:09,454 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:09,454 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=13, startTime=1731568629405; duration=0sec 2024-11-14T07:17:09,454 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:09,454 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:17:09,454 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:09,455 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:09,455 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:17:09,456 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:09,456 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/b4b92712867e4d47b5d20388df2aac08, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9aa1f2828b08423dbda22ebe1093d1d6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/824b80ddb26347839cc8d9af26e8772e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=36.7 K 2024-11-14T07:17:09,456 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b92712867e4d47b5d20388df2aac08, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731568627462 2024-11-14T07:17:09,457 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa1f2828b08423dbda22ebe1093d1d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731568627556 2024-11-14T07:17:09,458 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 824b80ddb26347839cc8d9af26e8772e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731568628195 2024-11-14T07:17:09,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742127_1303 (size=12301) 2024-11-14T07:17:09,473 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:09,473 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/9648ec71ffaf49a3838b4705b34e273d is 50, key is test_row_0/C:col10/1731568628216/Put/seqid=0 2024-11-14T07:17:09,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742128_1304 (size=13119) 2024-11-14T07:17:09,499 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/9648ec71ffaf49a3838b4705b34e273d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9648ec71ffaf49a3838b4705b34e273d 2024-11-14T07:17:09,505 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into 9648ec71ffaf49a3838b4705b34e273d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:09,505 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:09,506 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=13, startTime=1731568629405; duration=0sec 2024-11-14T07:17:09,506 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:09,506 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:17:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:17:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568689737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568689741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33484 deadline: 1731568689812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731568689812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,814 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,815 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33470 deadline: 1731568689821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,823 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in b5566aea29a31d45dee688596a38fef4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,835 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/dc365bae15fc40239c603aeac5127621 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/dc365bae15fc40239c603aeac5127621 2024-11-14T07:17:09,839 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into dc365bae15fc40239c603aeac5127621(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:09,839 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:09,839 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=13, startTime=1731568629405; duration=0sec 2024-11-14T07:17:09,839 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:09,839 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:17:09,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568689852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568689852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:09,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/92ff4dbafe48487496ea0344a441acc9 2024-11-14T07:17:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/09ca35d4fa3d4274bb065ad1e33ee6e6 is 50, key is test_row_0/B:col10/1731568628581/Put/seqid=0 2024-11-14T07:17:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742129_1305 (size=12301) 2024-11-14T07:17:10,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568690061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568690061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:17:10,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/09ca35d4fa3d4274bb065ad1e33ee6e6 2024-11-14T07:17:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/bdfc49290569452aa0d5288aa530cb7a is 50, key is test_row_0/C:col10/1731568628581/Put/seqid=0 2024-11-14T07:17:10,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742130_1306 (size=12301) 2024-11-14T07:17:10,311 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/bdfc49290569452aa0d5288aa530cb7a 2024-11-14T07:17:10,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/92ff4dbafe48487496ea0344a441acc9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/92ff4dbafe48487496ea0344a441acc9 2024-11-14T07:17:10,330 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/92ff4dbafe48487496ea0344a441acc9, entries=150, sequenceid=378, filesize=12.0 K 2024-11-14T07:17:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/09ca35d4fa3d4274bb065ad1e33ee6e6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/09ca35d4fa3d4274bb065ad1e33ee6e6 2024-11-14T07:17:10,336 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/09ca35d4fa3d4274bb065ad1e33ee6e6, entries=150, sequenceid=378, filesize=12.0 K 2024-11-14T07:17:10,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/bdfc49290569452aa0d5288aa530cb7a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/bdfc49290569452aa0d5288aa530cb7a 2024-11-14T07:17:10,342 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/bdfc49290569452aa0d5288aa530cb7a, entries=150, sequenceid=378, filesize=12.0 K 2024-11-14T07:17:10,343 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b5566aea29a31d45dee688596a38fef4 in 911ms, sequenceid=378, compaction requested=false 2024-11-14T07:17:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-14T07:17:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-14T07:17:10,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T07:17:10,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-14T07:17:10,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.3760 sec 2024-11-14T07:17:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:10,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:17:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:10,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/bc56789b010e4a38b818b00bd321a5ae is 50, key is test_row_0/A:col10/1731568629739/Put/seqid=0 2024-11-14T07:17:10,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742131_1307 (size=17181) 2024-11-14T07:17:10,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568690410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568690415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568690521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568690522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568690733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568690733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:10,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/bc56789b010e4a38b818b00bd321a5ae 2024-11-14T07:17:10,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/8a876649629d49a785f0d97716f1b7a4 is 50, key is test_row_0/B:col10/1731568629739/Put/seqid=0 2024-11-14T07:17:10,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742132_1308 (size=12301) 2024-11-14T07:17:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568691042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568691042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:17:11,133 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:11,135 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-14T07:17:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:17:11,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:11,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:11,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:11,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/8a876649629d49a785f0d97716f1b7a4 2024-11-14T07:17:11,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/06b47fda195f4418a2b18be4d963da22 is 50, key is test_row_0/C:col10/1731568629739/Put/seqid=0 2024-11-14T07:17:11,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742133_1309 (size=12301) 2024-11-14T07:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:17:11,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:17:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:17:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:17:11,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1731568691552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:11,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33510 deadline: 1731568691552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:11,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:11,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:11,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:17:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. as already flushing 2024-11-14T07:17:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:11,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/06b47fda195f4418a2b18be4d963da22 2024-11-14T07:17:11,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/bc56789b010e4a38b818b00bd321a5ae as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/bc56789b010e4a38b818b00bd321a5ae 2024-11-14T07:17:11,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/bc56789b010e4a38b818b00bd321a5ae, entries=250, sequenceid=395, filesize=16.8 K 2024-11-14T07:17:11,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/8a876649629d49a785f0d97716f1b7a4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8a876649629d49a785f0d97716f1b7a4 2024-11-14T07:17:11,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8a876649629d49a785f0d97716f1b7a4, entries=150, sequenceid=395, filesize=12.0 K 2024-11-14T07:17:11,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/06b47fda195f4418a2b18be4d963da22 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/06b47fda195f4418a2b18be4d963da22 2024-11-14T07:17:11,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/06b47fda195f4418a2b18be4d963da22, entries=150, sequenceid=395, filesize=12.0 K 2024-11-14T07:17:11,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b5566aea29a31d45dee688596a38fef4 in 1283ms, sequenceid=395, compaction requested=true 2024-11-14T07:17:11,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:11,657 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:11,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:11,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:11,658 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:11,658 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:11,658 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/A is initiating minor compaction (all files) 2024-11-14T07:17:11,658 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/A in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:11,659 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/dc365bae15fc40239c603aeac5127621, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/92ff4dbafe48487496ea0344a441acc9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/bc56789b010e4a38b818b00bd321a5ae] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=41.6 K 2024-11-14T07:17:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5566aea29a31d45dee688596a38fef4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:11,659 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc365bae15fc40239c603aeac5127621, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731568628195 2024-11-14T07:17:11,659 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92ff4dbafe48487496ea0344a441acc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731568628574 2024-11-14T07:17:11,660 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:11,660 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/B is initiating minor compaction (all files) 2024-11-14T07:17:11,660 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/B in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,660 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5efde1afc1b34da78e8c9194f53621a6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/09ca35d4fa3d4274bb065ad1e33ee6e6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8a876649629d49a785f0d97716f1b7a4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=36.8 K 2024-11-14T07:17:11,660 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc56789b010e4a38b818b00bd321a5ae, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731568629727 2024-11-14T07:17:11,661 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5efde1afc1b34da78e8c9194f53621a6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731568628195 2024-11-14T07:17:11,663 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 09ca35d4fa3d4274bb065ad1e33ee6e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731568628574 2024-11-14T07:17:11,663 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a876649629d49a785f0d97716f1b7a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731568629727 2024-11-14T07:17:11,683 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:11,683 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/179cb17937fa42758d7dc8b1f46445bd is 50, key is test_row_0/A:col10/1731568629739/Put/seqid=0 2024-11-14T07:17:11,694 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:11,694 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/23cf7505e09c4287802e95ff3f1b0aaa is 50, key is test_row_0/B:col10/1731568629739/Put/seqid=0 2024-11-14T07:17:11,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:17:11,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,752 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T07:17:11,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:11,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:11,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:11,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:11,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:11,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:17:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742134_1310 (size=13221) 2024-11-14T07:17:11,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742135_1311 (size=13221) 2024-11-14T07:17:11,781 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/23cf7505e09c4287802e95ff3f1b0aaa as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/23cf7505e09c4287802e95ff3f1b0aaa 2024-11-14T07:17:11,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/730799ee3e5240848aaacc6e0ae47ab8 is 50, key is test_row_0/A:col10/1731568630402/Put/seqid=0 2024-11-14T07:17:11,790 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/B of b5566aea29a31d45dee688596a38fef4 into 23cf7505e09c4287802e95ff3f1b0aaa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:11,790 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:11,790 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/B, priority=13, startTime=1731568631658; duration=0sec 2024-11-14T07:17:11,790 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:11,790 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:B 2024-11-14T07:17:11,790 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:11,791 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:11,791 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): b5566aea29a31d45dee688596a38fef4/C is initiating minor compaction (all files) 2024-11-14T07:17:11,791 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5566aea29a31d45dee688596a38fef4/C in TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:11,791 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9648ec71ffaf49a3838b4705b34e273d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/bdfc49290569452aa0d5288aa530cb7a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/06b47fda195f4418a2b18be4d963da22] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp, totalSize=36.8 K 2024-11-14T07:17:11,792 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9648ec71ffaf49a3838b4705b34e273d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731568628195 2024-11-14T07:17:11,792 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfc49290569452aa0d5288aa530cb7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731568628574 2024-11-14T07:17:11,792 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b47fda195f4418a2b18be4d963da22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731568629727 2024-11-14T07:17:11,816 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5566aea29a31d45dee688596a38fef4#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:11,817 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e28aadc59bea4d478f19122d19e33472 is 50, key is test_row_0/C:col10/1731568629739/Put/seqid=0 2024-11-14T07:17:11,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742136_1312 (size=12301) 2024-11-14T07:17:11,840 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/730799ee3e5240848aaacc6e0ae47ab8 2024-11-14T07:17:11,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/62ce15e9588a4704b547d6e0ecb346fd is 50, key is test_row_0/B:col10/1731568630402/Put/seqid=0 2024-11-14T07:17:11,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742137_1313 (size=13221) 2024-11-14T07:17:11,873 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/e28aadc59bea4d478f19122d19e33472 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e28aadc59bea4d478f19122d19e33472 2024-11-14T07:17:11,880 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/C of b5566aea29a31d45dee688596a38fef4 into e28aadc59bea4d478f19122d19e33472(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:11,880 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:11,880 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/C, priority=13, startTime=1731568631659; duration=0sec 2024-11-14T07:17:11,880 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:11,880 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:C 2024-11-14T07:17:11,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742138_1314 (size=12301) 2024-11-14T07:17:12,177 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/179cb17937fa42758d7dc8b1f46445bd as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/179cb17937fa42758d7dc8b1f46445bd 2024-11-14T07:17:12,184 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5566aea29a31d45dee688596a38fef4/A of b5566aea29a31d45dee688596a38fef4 into 179cb17937fa42758d7dc8b1f46445bd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:12,184 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:12,184 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., storeName=b5566aea29a31d45dee688596a38fef4/A, priority=13, startTime=1731568631657; duration=0sec 2024-11-14T07:17:12,184 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:12,184 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5566aea29a31d45dee688596a38fef4:A 2024-11-14T07:17:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:17:12,288 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/62ce15e9588a4704b547d6e0ecb346fd 2024-11-14T07:17:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/dcb4d9bddf874e41a1e0f9beb2103e54 is 50, key is test_row_0/C:col10/1731568630402/Put/seqid=0 2024-11-14T07:17:12,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742139_1315 (size=12301) 2024-11-14T07:17:12,301 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/dcb4d9bddf874e41a1e0f9beb2103e54 2024-11-14T07:17:12,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/730799ee3e5240848aaacc6e0ae47ab8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/730799ee3e5240848aaacc6e0ae47ab8 2024-11-14T07:17:12,309 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/730799ee3e5240848aaacc6e0ae47ab8, entries=150, sequenceid=413, filesize=12.0 K 2024-11-14T07:17:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/62ce15e9588a4704b547d6e0ecb346fd as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/62ce15e9588a4704b547d6e0ecb346fd 2024-11-14T07:17:12,320 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/62ce15e9588a4704b547d6e0ecb346fd, entries=150, sequenceid=413, filesize=12.0 K 2024-11-14T07:17:12,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/dcb4d9bddf874e41a1e0f9beb2103e54 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/dcb4d9bddf874e41a1e0f9beb2103e54 2024-11-14T07:17:12,324 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/dcb4d9bddf874e41a1e0f9beb2103e54, entries=150, sequenceid=413, filesize=12.0 K 2024-11-14T07:17:12,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for b5566aea29a31d45dee688596a38fef4 in 573ms, sequenceid=413, compaction requested=false 2024-11-14T07:17:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for b5566aea29a31d45dee688596a38fef4: 2024-11-14T07:17:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-14T07:17:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-14T07:17:12,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T07:17:12,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1890 sec 2024-11-14T07:17:12,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1940 sec 2024-11-14T07:17:12,567 INFO [Thread-1087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1087. 2024-11-14T07:17:12,567 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-14T07:17:12,567 DEBUG [Thread-1087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:12,567 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:12,567 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,567 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,567 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,567 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-14T07:17:12,567 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:12,567 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,567 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:12,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:12,568 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-14T07:17:12,568 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:12,568 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,568 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:12,568 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:12,569 INFO [Thread-1085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1085. 2024-11-14T07:17:12,569 DEBUG [Thread-1085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:12,569 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,569 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:12,573 INFO [Thread-1074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1074. 2024-11-14T07:17:12,573 INFO [Thread-1076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1076. 2024-11-14T07:17:12,573 DEBUG [Thread-1074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:12,573 DEBUG [Thread-1076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:12,573 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,573 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,574 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,574 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:12,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:12,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:17:13,282 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:18,497 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:17:19,874 INFO [Thread-1078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1078. 2024-11-14T07:17:19,874 DEBUG [Thread-1078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:19,874 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,874 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:19,883 INFO [Thread-1072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1072. 2024-11-14T07:17:19,883 DEBUG [Thread-1072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:19,883 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,883 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:19,894 INFO [Thread-1070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1070. 2024-11-14T07:17:19,894 DEBUG [Thread-1070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:19,894 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,894 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3400 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3284 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3253 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3392 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3295 2024-11-14T07:17:19,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:17:19,894 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:17:19,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:17:19,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:19,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:19,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:17:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:17:19,898 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568639898"}]},"ts":"1731568639898"} 2024-11-14T07:17:19,899 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:17:19,899 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:17:19,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:17:19,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, UNASSIGN}] 2024-11-14T07:17:19,902 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, UNASSIGN 2024-11-14T07:17:19,902 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b5566aea29a31d45dee688596a38fef4, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:19,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, UNASSIGN because future has completed 2024-11-14T07:17:19,904 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:17:19,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:17:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:17:20,056 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:20,056 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:17:20,056 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing b5566aea29a31d45dee688596a38fef4, disabling compactions & flushes 2024-11-14T07:17:20,056 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:20,056 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:20,056 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. after waiting 0 ms 2024-11-14T07:17:20,057 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:20,057 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing b5566aea29a31d45dee688596a38fef4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T07:17:20,057 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=A 2024-11-14T07:17:20,057 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:20,057 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=B 2024-11-14T07:17:20,057 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:20,057 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5566aea29a31d45dee688596a38fef4, store=C 2024-11-14T07:17:20,057 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:20,061 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/61a472e048ff4927825df59d1fb6bbf9 is 50, key is test_row_0/A:col10/1731568639892/Put/seqid=0 2024-11-14T07:17:20,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742140_1316 (size=9857) 2024-11-14T07:17:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:17:20,466 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/61a472e048ff4927825df59d1fb6bbf9 2024-11-14T07:17:20,476 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/03b320f8efce491fb4abd3ae40d53d45 is 50, key is test_row_0/B:col10/1731568639892/Put/seqid=0 2024-11-14T07:17:20,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742141_1317 (size=9857) 2024-11-14T07:17:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:17:20,880 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/03b320f8efce491fb4abd3ae40d53d45 2024-11-14T07:17:20,891 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/de875533b5a24b8d8e95a50f6f34f25c is 50, key is test_row_0/C:col10/1731568639892/Put/seqid=0 2024-11-14T07:17:20,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742142_1318 (size=9857) 2024-11-14T07:17:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:17:21,296 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/de875533b5a24b8d8e95a50f6f34f25c 2024-11-14T07:17:21,305 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/A/61a472e048ff4927825df59d1fb6bbf9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/61a472e048ff4927825df59d1fb6bbf9 2024-11-14T07:17:21,312 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/61a472e048ff4927825df59d1fb6bbf9, entries=100, sequenceid=424, filesize=9.6 K 2024-11-14T07:17:21,313 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/B/03b320f8efce491fb4abd3ae40d53d45 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/03b320f8efce491fb4abd3ae40d53d45 2024-11-14T07:17:21,318 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/03b320f8efce491fb4abd3ae40d53d45, entries=100, sequenceid=424, filesize=9.6 K 2024-11-14T07:17:21,319 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/.tmp/C/de875533b5a24b8d8e95a50f6f34f25c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/de875533b5a24b8d8e95a50f6f34f25c 2024-11-14T07:17:21,323 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/de875533b5a24b8d8e95a50f6f34f25c, entries=100, sequenceid=424, filesize=9.6 K 2024-11-14T07:17:21,324 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b5566aea29a31d45dee688596a38fef4 in 1267ms, sequenceid=424, compaction requested=true 2024-11-14T07:17:21,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/53043f98824e4d52aeba1ef739d523c3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/676b5071f95c457a9169e8cc944f37c7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/5db818b8107f496b8a1937a1dc48496f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f9e013fef57d4b059ab93c6e1a28dbb0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/47b900ff02964c559e5b4dffa47826f9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07c5c547d6d4c28b2c25d7bf4a91739, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/eaf43deb11454edb9885e314cca2b622, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/0399afccdea34d75b6e55a69b1f35e9c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/e68a9d2736e44561980e67a164ea9927, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/7a2c4f40b74c46a992614b561055866a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/c849fab52fa54efdbc437c9a8944713f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/fcd643104e43443fab04be2baecd2129, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/af96222db37d47ad98e6fccb57a01250, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/d11e40fd5e354898b27d99118f14d660, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/152350dc5a614463ae1d1044282cd91a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/6f4efc9361ed4980b3cb022a2267f640, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/20056a3755434e7b8fcbbeac0ce75adb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/34b767f883244e5b83bb566a4356958c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/29e6743df68b42c9a1aecc5b257b4646, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/28437756d19547e4b69f110ad401ffc7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/1bacd38ce4454f1da4761208af8d2ba2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/a3665e01e7be4c8693605a18b7fdb9e7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/9a6bd5a739d94614a9ba88944d2ae26e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07860c03b164e5d8e0ca27eb2ebacce, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/dc365bae15fc40239c603aeac5127621, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/92ff4dbafe48487496ea0344a441acc9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/bc56789b010e4a38b818b00bd321a5ae] to archive 2024-11-14T07:17:21,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:17:21,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/53043f98824e4d52aeba1ef739d523c3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/53043f98824e4d52aeba1ef739d523c3 2024-11-14T07:17:21,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/676b5071f95c457a9169e8cc944f37c7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/676b5071f95c457a9169e8cc944f37c7 2024-11-14T07:17:21,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/5db818b8107f496b8a1937a1dc48496f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/5db818b8107f496b8a1937a1dc48496f 2024-11-14T07:17:21,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f9e013fef57d4b059ab93c6e1a28dbb0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f9e013fef57d4b059ab93c6e1a28dbb0 2024-11-14T07:17:21,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/47b900ff02964c559e5b4dffa47826f9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/47b900ff02964c559e5b4dffa47826f9 2024-11-14T07:17:21,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07c5c547d6d4c28b2c25d7bf4a91739 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07c5c547d6d4c28b2c25d7bf4a91739 2024-11-14T07:17:21,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/eaf43deb11454edb9885e314cca2b622 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/eaf43deb11454edb9885e314cca2b622 2024-11-14T07:17:21,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/0399afccdea34d75b6e55a69b1f35e9c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/0399afccdea34d75b6e55a69b1f35e9c 2024-11-14T07:17:21,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/e68a9d2736e44561980e67a164ea9927 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/e68a9d2736e44561980e67a164ea9927 2024-11-14T07:17:21,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/7a2c4f40b74c46a992614b561055866a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/7a2c4f40b74c46a992614b561055866a 2024-11-14T07:17:21,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/c849fab52fa54efdbc437c9a8944713f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/c849fab52fa54efdbc437c9a8944713f 2024-11-14T07:17:21,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/fcd643104e43443fab04be2baecd2129 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/fcd643104e43443fab04be2baecd2129 2024-11-14T07:17:21,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/af96222db37d47ad98e6fccb57a01250 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/af96222db37d47ad98e6fccb57a01250 2024-11-14T07:17:21,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/d11e40fd5e354898b27d99118f14d660 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/d11e40fd5e354898b27d99118f14d660 2024-11-14T07:17:21,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/152350dc5a614463ae1d1044282cd91a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/152350dc5a614463ae1d1044282cd91a 2024-11-14T07:17:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/6f4efc9361ed4980b3cb022a2267f640 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/6f4efc9361ed4980b3cb022a2267f640 2024-11-14T07:17:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/20056a3755434e7b8fcbbeac0ce75adb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/20056a3755434e7b8fcbbeac0ce75adb 2024-11-14T07:17:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/34b767f883244e5b83bb566a4356958c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/34b767f883244e5b83bb566a4356958c 2024-11-14T07:17:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/29e6743df68b42c9a1aecc5b257b4646 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/29e6743df68b42c9a1aecc5b257b4646 2024-11-14T07:17:21,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/28437756d19547e4b69f110ad401ffc7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/28437756d19547e4b69f110ad401ffc7 2024-11-14T07:17:21,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/1bacd38ce4454f1da4761208af8d2ba2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/1bacd38ce4454f1da4761208af8d2ba2 2024-11-14T07:17:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/a3665e01e7be4c8693605a18b7fdb9e7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/a3665e01e7be4c8693605a18b7fdb9e7 2024-11-14T07:17:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/9a6bd5a739d94614a9ba88944d2ae26e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/9a6bd5a739d94614a9ba88944d2ae26e 2024-11-14T07:17:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07860c03b164e5d8e0ca27eb2ebacce to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/f07860c03b164e5d8e0ca27eb2ebacce 2024-11-14T07:17:21,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/dc365bae15fc40239c603aeac5127621 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/dc365bae15fc40239c603aeac5127621 2024-11-14T07:17:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/92ff4dbafe48487496ea0344a441acc9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/92ff4dbafe48487496ea0344a441acc9 2024-11-14T07:17:21,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/bc56789b010e4a38b818b00bd321a5ae to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/bc56789b010e4a38b818b00bd321a5ae 2024-11-14T07:17:21,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/f7e4e215150c47dc96aecc6c8ca06186, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a59b175384d345cbb442faac190790c1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8e6ca48c463c4f86bcd4405315024391, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/d31e123189794e0799bf48bb77dac293, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a000211650db491b9ee1338be07dfeca, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6820a7ac0ce94b51b2fdba93ec5ad735, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/c0afffd0f0d949ec89e2a101502b4a4b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/fc033a1ecd034c0fb357f52da18fcbf8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6b894426e65e4bfca2cdfb8f4eb87247, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/356be9550aba41dcb8b39f0aa57c946b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4856389d20ac45908c957288ddad816d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4acf2ddfc7884e96bbcf89af9a85f6f0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/b28e6290606a4041890ede9e9af5dd64, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5201f8a79f674f6d926d52818e6348e0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5ab786604f274aa68013c6fc7ac2771f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5a5914bd06164003a4715dc7427048f4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/0e6425c7fe1d411fbd28dde980647c7c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/65ba738c64974f61a1042ad2aeedc972, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/339875a0a7b14dd1bf2a62de086bbf8a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/36374e2752ba4e65b2da265f473e2704, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/de6957d37f5148baba30ac6a04ec3a5e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/9b120132b383416981ecbd8623368ade, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/cd45ad9131dd4f9fb8c00f5a570db41d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5efde1afc1b34da78e8c9194f53621a6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/2fa7f4364bde41cc8249a6e9f0fd2638, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/09ca35d4fa3d4274bb065ad1e33ee6e6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8a876649629d49a785f0d97716f1b7a4] to archive 2024-11-14T07:17:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:17:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/f7e4e215150c47dc96aecc6c8ca06186 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/f7e4e215150c47dc96aecc6c8ca06186 2024-11-14T07:17:21,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a59b175384d345cbb442faac190790c1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a59b175384d345cbb442faac190790c1 2024-11-14T07:17:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8e6ca48c463c4f86bcd4405315024391 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8e6ca48c463c4f86bcd4405315024391 2024-11-14T07:17:21,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/d31e123189794e0799bf48bb77dac293 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/d31e123189794e0799bf48bb77dac293 2024-11-14T07:17:21,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a000211650db491b9ee1338be07dfeca to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/a000211650db491b9ee1338be07dfeca 2024-11-14T07:17:21,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6820a7ac0ce94b51b2fdba93ec5ad735 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6820a7ac0ce94b51b2fdba93ec5ad735 2024-11-14T07:17:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/c0afffd0f0d949ec89e2a101502b4a4b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/c0afffd0f0d949ec89e2a101502b4a4b 2024-11-14T07:17:21,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/fc033a1ecd034c0fb357f52da18fcbf8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/fc033a1ecd034c0fb357f52da18fcbf8 2024-11-14T07:17:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6b894426e65e4bfca2cdfb8f4eb87247 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/6b894426e65e4bfca2cdfb8f4eb87247 2024-11-14T07:17:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/356be9550aba41dcb8b39f0aa57c946b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/356be9550aba41dcb8b39f0aa57c946b 2024-11-14T07:17:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4856389d20ac45908c957288ddad816d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4856389d20ac45908c957288ddad816d 2024-11-14T07:17:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4acf2ddfc7884e96bbcf89af9a85f6f0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/4acf2ddfc7884e96bbcf89af9a85f6f0 2024-11-14T07:17:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/b28e6290606a4041890ede9e9af5dd64 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/b28e6290606a4041890ede9e9af5dd64 2024-11-14T07:17:21,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5201f8a79f674f6d926d52818e6348e0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5201f8a79f674f6d926d52818e6348e0 2024-11-14T07:17:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5ab786604f274aa68013c6fc7ac2771f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5ab786604f274aa68013c6fc7ac2771f 2024-11-14T07:17:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5a5914bd06164003a4715dc7427048f4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5a5914bd06164003a4715dc7427048f4 2024-11-14T07:17:21,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/0e6425c7fe1d411fbd28dde980647c7c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/0e6425c7fe1d411fbd28dde980647c7c 2024-11-14T07:17:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/65ba738c64974f61a1042ad2aeedc972 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/65ba738c64974f61a1042ad2aeedc972 2024-11-14T07:17:21,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/339875a0a7b14dd1bf2a62de086bbf8a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/339875a0a7b14dd1bf2a62de086bbf8a 2024-11-14T07:17:21,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/36374e2752ba4e65b2da265f473e2704 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/36374e2752ba4e65b2da265f473e2704 2024-11-14T07:17:21,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/de6957d37f5148baba30ac6a04ec3a5e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/de6957d37f5148baba30ac6a04ec3a5e 2024-11-14T07:17:21,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/9b120132b383416981ecbd8623368ade to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/9b120132b383416981ecbd8623368ade 2024-11-14T07:17:21,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/cd45ad9131dd4f9fb8c00f5a570db41d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/cd45ad9131dd4f9fb8c00f5a570db41d 2024-11-14T07:17:21,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5efde1afc1b34da78e8c9194f53621a6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/5efde1afc1b34da78e8c9194f53621a6 2024-11-14T07:17:21,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/2fa7f4364bde41cc8249a6e9f0fd2638 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/2fa7f4364bde41cc8249a6e9f0fd2638 2024-11-14T07:17:21,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/09ca35d4fa3d4274bb065ad1e33ee6e6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/09ca35d4fa3d4274bb065ad1e33ee6e6 2024-11-14T07:17:21,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8a876649629d49a785f0d97716f1b7a4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/8a876649629d49a785f0d97716f1b7a4 2024-11-14T07:17:21,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/6133c6df93dd4a4dbc456e9988959d2d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5a2c2eff1ee9447e8c84cfef9a80b28c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2975148177414ee8aa3c5c469170bcb5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/0674780fb3ce4852bc2080344c9be868, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/387f69eff9e8421d9dfa12933bfd6551, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/00c8b614b3b44e58b80dee81579e7c72, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/c2755e908bdc4173aefaedd14dfa7b73, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e271cf630e2441b3b71aa4f7d249ec2e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/a86f648d650d46c19966c61b551a6a06, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/d35c637d7d3a4349abe9a4a992ddbdff, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/1099b86c50d844a499bdac3c87b3c5c8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e7645c1540954b7297baa06c98f7976c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/01370b176ecc4a989a2b46f8ad3645f7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/67fe63afe4da447b9cdbc2d45fc4111f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/aa5ec563c6aa4552b8b3351478bb63f4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5411bf7dd82e45b899d6d5979dbdb13f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/31b29ae73aeb420ea7a0a65346b8ff34, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2d3e0dc33f9246c5a53b6b0da32f9fe2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2bbad79ec3b14e4280c5c10a2b393751, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2a6be27867dd43ada00bb15765779532, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/b4b92712867e4d47b5d20388df2aac08, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/ee9f96b813ed42d9a7b636e85905186d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9aa1f2828b08423dbda22ebe1093d1d6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9648ec71ffaf49a3838b4705b34e273d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/824b80ddb26347839cc8d9af26e8772e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/bdfc49290569452aa0d5288aa530cb7a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/06b47fda195f4418a2b18be4d963da22] to archive 2024-11-14T07:17:21,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:17:21,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/6133c6df93dd4a4dbc456e9988959d2d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/6133c6df93dd4a4dbc456e9988959d2d 2024-11-14T07:17:21,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5a2c2eff1ee9447e8c84cfef9a80b28c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5a2c2eff1ee9447e8c84cfef9a80b28c 2024-11-14T07:17:21,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2975148177414ee8aa3c5c469170bcb5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2975148177414ee8aa3c5c469170bcb5 2024-11-14T07:17:21,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/0674780fb3ce4852bc2080344c9be868 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/0674780fb3ce4852bc2080344c9be868 2024-11-14T07:17:21,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/387f69eff9e8421d9dfa12933bfd6551 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/387f69eff9e8421d9dfa12933bfd6551 2024-11-14T07:17:21,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/00c8b614b3b44e58b80dee81579e7c72 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/00c8b614b3b44e58b80dee81579e7c72 2024-11-14T07:17:21,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/c2755e908bdc4173aefaedd14dfa7b73 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/c2755e908bdc4173aefaedd14dfa7b73 2024-11-14T07:17:21,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e271cf630e2441b3b71aa4f7d249ec2e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e271cf630e2441b3b71aa4f7d249ec2e 2024-11-14T07:17:21,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/a86f648d650d46c19966c61b551a6a06 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/a86f648d650d46c19966c61b551a6a06 2024-11-14T07:17:21,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/d35c637d7d3a4349abe9a4a992ddbdff to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/d35c637d7d3a4349abe9a4a992ddbdff 2024-11-14T07:17:21,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/1099b86c50d844a499bdac3c87b3c5c8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/1099b86c50d844a499bdac3c87b3c5c8 2024-11-14T07:17:21,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e7645c1540954b7297baa06c98f7976c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e7645c1540954b7297baa06c98f7976c 2024-11-14T07:17:21,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/01370b176ecc4a989a2b46f8ad3645f7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/01370b176ecc4a989a2b46f8ad3645f7 2024-11-14T07:17:21,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/67fe63afe4da447b9cdbc2d45fc4111f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/67fe63afe4da447b9cdbc2d45fc4111f 2024-11-14T07:17:21,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/aa5ec563c6aa4552b8b3351478bb63f4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/aa5ec563c6aa4552b8b3351478bb63f4 2024-11-14T07:17:21,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5411bf7dd82e45b899d6d5979dbdb13f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/5411bf7dd82e45b899d6d5979dbdb13f 2024-11-14T07:17:21,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/31b29ae73aeb420ea7a0a65346b8ff34 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/31b29ae73aeb420ea7a0a65346b8ff34 2024-11-14T07:17:21,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2d3e0dc33f9246c5a53b6b0da32f9fe2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2d3e0dc33f9246c5a53b6b0da32f9fe2 2024-11-14T07:17:21,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2bbad79ec3b14e4280c5c10a2b393751 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2bbad79ec3b14e4280c5c10a2b393751 2024-11-14T07:17:21,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2a6be27867dd43ada00bb15765779532 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/2a6be27867dd43ada00bb15765779532 2024-11-14T07:17:21,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/b4b92712867e4d47b5d20388df2aac08 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/b4b92712867e4d47b5d20388df2aac08 2024-11-14T07:17:21,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/ee9f96b813ed42d9a7b636e85905186d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/ee9f96b813ed42d9a7b636e85905186d 2024-11-14T07:17:21,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9aa1f2828b08423dbda22ebe1093d1d6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9aa1f2828b08423dbda22ebe1093d1d6 2024-11-14T07:17:21,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9648ec71ffaf49a3838b4705b34e273d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/9648ec71ffaf49a3838b4705b34e273d 2024-11-14T07:17:21,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/824b80ddb26347839cc8d9af26e8772e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/824b80ddb26347839cc8d9af26e8772e 2024-11-14T07:17:21,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/bdfc49290569452aa0d5288aa530cb7a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/bdfc49290569452aa0d5288aa530cb7a 2024-11-14T07:17:21,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/06b47fda195f4418a2b18be4d963da22 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/06b47fda195f4418a2b18be4d963da22 2024-11-14T07:17:21,425 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/recovered.edits/427.seqid, newMaxSeqId=427, maxSeqId=1 2024-11-14T07:17:21,426 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. 2024-11-14T07:17:21,426 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for b5566aea29a31d45dee688596a38fef4: Waiting for close lock at 1731568640056Running coprocessor pre-close hooks at 1731568640056Disabling compacts and flushes for region at 1731568640056Disabling writes for close at 1731568640056Obtaining lock to block concurrent updates at 1731568640057 (+1 ms)Preparing flush snapshotting stores in b5566aea29a31d45dee688596a38fef4 at 1731568640057Finished memstore snapshotting TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731568640057Flushing stores of TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4. at 1731568640058 (+1 ms)Flushing b5566aea29a31d45dee688596a38fef4/A: creating writer at 1731568640058Flushing b5566aea29a31d45dee688596a38fef4/A: appending metadata at 1731568640061 (+3 ms)Flushing b5566aea29a31d45dee688596a38fef4/A: closing flushed file at 1731568640061Flushing b5566aea29a31d45dee688596a38fef4/B: creating writer at 1731568640474 (+413 ms)Flushing b5566aea29a31d45dee688596a38fef4/B: appending metadata at 1731568640476 (+2 ms)Flushing b5566aea29a31d45dee688596a38fef4/B: closing flushed file at 1731568640476Flushing b5566aea29a31d45dee688596a38fef4/C: creating writer at 1731568640888 (+412 ms)Flushing b5566aea29a31d45dee688596a38fef4/C: appending metadata at 1731568640891 (+3 ms)Flushing b5566aea29a31d45dee688596a38fef4/C: closing flushed file at 1731568640891Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4795394c: reopening flushed file at 1731568641304 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ccf41fe: reopening flushed file at 1731568641312 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a75da27: reopening flushed file at 1731568641318 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b5566aea29a31d45dee688596a38fef4 in 1267ms, sequenceid=424, compaction requested=true at 1731568641324 (+6 ms)Writing region close event to WAL at 1731568641422 (+98 ms)Running coprocessor post-close hooks at 1731568641425 (+3 ms)Closed at 1731568641426 (+1 ms) 2024-11-14T07:17:21,428 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:21,428 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b5566aea29a31d45dee688596a38fef4, regionState=CLOSED 2024-11-14T07:17:21,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:17:21,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-14T07:17:21,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure b5566aea29a31d45dee688596a38fef4, server=c7aa6e2790b3,33845,1731568550163 in 1.5270 sec 2024-11-14T07:17:21,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-14T07:17:21,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5566aea29a31d45dee688596a38fef4, UNASSIGN in 1.5310 sec 2024-11-14T07:17:21,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T07:17:21,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5340 sec 2024-11-14T07:17:21,436 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568641436"}]},"ts":"1731568641436"} 2024-11-14T07:17:21,437 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:17:21,437 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:17:21,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5430 sec 2024-11-14T07:17:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:17:22,042 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:22,042 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:17:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:22,044 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:22,045 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T07:17:22,046 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:22,049 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/recovered.edits] 2024-11-14T07:17:22,051 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/179cb17937fa42758d7dc8b1f46445bd to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/179cb17937fa42758d7dc8b1f46445bd 2024-11-14T07:17:22,053 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/61a472e048ff4927825df59d1fb6bbf9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/61a472e048ff4927825df59d1fb6bbf9 2024-11-14T07:17:22,054 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/730799ee3e5240848aaacc6e0ae47ab8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/A/730799ee3e5240848aaacc6e0ae47ab8 2024-11-14T07:17:22,056 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/03b320f8efce491fb4abd3ae40d53d45 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/03b320f8efce491fb4abd3ae40d53d45 2024-11-14T07:17:22,057 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/23cf7505e09c4287802e95ff3f1b0aaa to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/23cf7505e09c4287802e95ff3f1b0aaa 2024-11-14T07:17:22,059 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/62ce15e9588a4704b547d6e0ecb346fd to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/B/62ce15e9588a4704b547d6e0ecb346fd 2024-11-14T07:17:22,061 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/dcb4d9bddf874e41a1e0f9beb2103e54 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/dcb4d9bddf874e41a1e0f9beb2103e54 2024-11-14T07:17:22,062 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/de875533b5a24b8d8e95a50f6f34f25c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/de875533b5a24b8d8e95a50f6f34f25c 2024-11-14T07:17:22,063 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e28aadc59bea4d478f19122d19e33472 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/C/e28aadc59bea4d478f19122d19e33472 2024-11-14T07:17:22,067 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/recovered.edits/427.seqid to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4/recovered.edits/427.seqid 2024-11-14T07:17:22,067 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/b5566aea29a31d45dee688596a38fef4 2024-11-14T07:17:22,067 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:17:22,070 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:22,076 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:17:22,078 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:17:22,080 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:22,080 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:17:22,080 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731568642080"}]},"ts":"9223372036854775807"} 2024-11-14T07:17:22,082 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:17:22,082 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b5566aea29a31d45dee688596a38fef4, NAME => 'TestAcidGuarantees,,1731568610353.b5566aea29a31d45dee688596a38fef4.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:17:22,082 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:17:22,083 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731568642083"}]},"ts":"9223372036854775807"} 2024-11-14T07:17:22,085 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:17:22,086 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:22,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-14T07:17:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T07:17:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:17:22,155 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:22,163 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 251), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=452 (was 466), ProcessCount=11 (was 11), AvailableMemoryMB=6151 (was 5471) - AvailableMemoryMB LEAK? - 2024-11-14T07:17:22,174 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=452, ProcessCount=11, AvailableMemoryMB=6150 2024-11-14T07:17:22,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:17:22,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:17:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:22,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:17:22,178 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:22,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-14T07:17:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T07:17:22,179 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:17:22,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742143_1319 (size=1061) 2024-11-14T07:17:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T07:17:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T07:17:22,587 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:17:22,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742144_1320 (size=53) 2024-11-14T07:17:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T07:17:22,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:17:22,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 17b20244019df3f0f04e41107ed48efc, disabling compactions & flushes 2024-11-14T07:17:22,996 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:22,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:22,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. after waiting 0 ms 2024-11-14T07:17:22,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:22,996 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:22,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17b20244019df3f0f04e41107ed48efc: Waiting for close lock at 1731568642996Disabling compacts and flushes for region at 1731568642996Disabling writes for close at 1731568642996Writing region close event to WAL at 1731568642996Closed at 1731568642996 2024-11-14T07:17:22,997 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:17:22,997 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731568642997"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731568642997"}]},"ts":"1731568642997"} 2024-11-14T07:17:22,998 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:17:22,999 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:17:22,999 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568642999"}]},"ts":"1731568642999"} 2024-11-14T07:17:23,000 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:17:23,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, ASSIGN}] 2024-11-14T07:17:23,001 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, ASSIGN 2024-11-14T07:17:23,002 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=false 2024-11-14T07:17:23,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:23,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, ASSIGN because future has completed 2024-11-14T07:17:23,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:17:23,312 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:23,312 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:17:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T07:17:23,312 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,313 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:17:23,313 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,313 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,315 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,317 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:23,318 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b20244019df3f0f04e41107ed48efc columnFamilyName A 2024-11-14T07:17:23,318 DEBUG [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:23,318 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(327): Store=17b20244019df3f0f04e41107ed48efc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:23,319 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,320 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:23,320 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b20244019df3f0f04e41107ed48efc columnFamilyName B 2024-11-14T07:17:23,320 DEBUG [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:23,321 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(327): Store=17b20244019df3f0f04e41107ed48efc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:23,321 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,323 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:23,323 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b20244019df3f0f04e41107ed48efc columnFamilyName C 2024-11-14T07:17:23,323 DEBUG [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:23,324 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(327): Store=17b20244019df3f0f04e41107ed48efc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:23,324 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:23,324 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,325 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,326 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,327 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,327 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,328 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:17:23,329 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,332 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:17:23,333 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 17b20244019df3f0f04e41107ed48efc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67201565, jitterRate=0.0013813525438308716}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:17:23,334 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:23,335 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 17b20244019df3f0f04e41107ed48efc: Running coprocessor pre-open hook at 1731568643313Writing region info on filesystem at 1731568643313Initializing all the Stores at 1731568643314 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568643314Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568643315 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568643315Cleaning up temporary data from old regions at 1731568643327 (+12 ms)Running coprocessor post-open hooks at 1731568643334 (+7 ms)Region opened successfully at 1731568643335 (+1 ms) 2024-11-14T07:17:23,336 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., pid=89, masterSystemTime=1731568643306 2024-11-14T07:17:23,338 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:23,338 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:23,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:23,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:17:23,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-14T07:17:23,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 in 186 msec 2024-11-14T07:17:23,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-14T07:17:23,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, ASSIGN in 342 msec 2024-11-14T07:17:23,345 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:17:23,345 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568643345"}]},"ts":"1731568643345"} 2024-11-14T07:17:23,346 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:17:23,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:17:23,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-14T07:17:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T07:17:24,323 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:24,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:24,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:24,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:24,329 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4500efba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:24,331 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca74d07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:24,332 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:24,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:24,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:17:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,37073,1731568549456 2024-11-14T07:17:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@674f6a05 2024-11-14T07:17:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:17:24,337 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:17:24,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:17:24,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:17:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:24,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742145_1321 (size=1097) 2024-11-14T07:17:24,752 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T07:17:24,752 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T07:17:24,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:17:24,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, REOPEN/MOVE}] 2024-11-14T07:17:24,761 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, REOPEN/MOVE 2024-11-14T07:17:24,762 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:24,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, REOPEN/MOVE because future has completed 2024-11-14T07:17:24,763 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:17:24,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:17:24,916 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:24,916 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:17:24,916 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 17b20244019df3f0f04e41107ed48efc, disabling compactions & flushes 2024-11-14T07:17:24,916 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:24,916 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:24,916 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. after waiting 0 ms 2024-11-14T07:17:24,917 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:24,924 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T07:17:24,925 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:24,925 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 17b20244019df3f0f04e41107ed48efc: Waiting for close lock at 1731568644916Running coprocessor pre-close hooks at 1731568644916Disabling compacts and flushes for region at 1731568644916Disabling writes for close at 1731568644917 (+1 ms)Writing region close event to WAL at 1731568644917Running coprocessor post-close hooks at 1731568644925 (+8 ms)Closed at 1731568644925 2024-11-14T07:17:24,925 WARN [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 17b20244019df3f0f04e41107ed48efc to self. 2024-11-14T07:17:24,929 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:24,930 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=CLOSED 2024-11-14T07:17:24,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:17:24,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-14T07:17:24,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 in 169 msec 2024-11-14T07:17:24,936 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, REOPEN/MOVE; state=CLOSED, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=true 2024-11-14T07:17:25,087 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, REOPEN/MOVE because future has completed 2024-11-14T07:17:25,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:17:25,247 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,247 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:17:25,248 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,248 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:17:25,248 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,248 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,251 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,253 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:25,254 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b20244019df3f0f04e41107ed48efc columnFamilyName A 2024-11-14T07:17:25,256 DEBUG [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:25,256 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(327): Store=17b20244019df3f0f04e41107ed48efc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:25,257 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,257 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:25,257 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b20244019df3f0f04e41107ed48efc columnFamilyName B 2024-11-14T07:17:25,257 DEBUG [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:25,258 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(327): Store=17b20244019df3f0f04e41107ed48efc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:25,258 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,258 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:25,259 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b20244019df3f0f04e41107ed48efc columnFamilyName C 2024-11-14T07:17:25,259 DEBUG [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:25,259 INFO [StoreOpener-17b20244019df3f0f04e41107ed48efc-1 {}] regionserver.HStore(327): Store=17b20244019df3f0f04e41107ed48efc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:25,259 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,259 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,260 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,261 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,262 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,262 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,262 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:17:25,263 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,264 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 17b20244019df3f0f04e41107ed48efc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72152101, jitterRate=0.07515008747577667}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:17:25,264 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,265 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 17b20244019df3f0f04e41107ed48efc: Running coprocessor pre-open hook at 1731568645249Writing region info on filesystem at 1731568645249Initializing all the Stores at 1731568645250 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568645250Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568645251 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568645251Cleaning up temporary data from old regions at 1731568645262 (+11 ms)Running coprocessor post-open hooks at 1731568645264 (+2 ms)Region opened successfully at 1731568645265 (+1 ms) 2024-11-14T07:17:25,265 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., pid=94, masterSystemTime=1731568645240 2024-11-14T07:17:25,267 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,267 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,267 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=OPEN, openSeqNum=5, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:17:25,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-14T07:17:25,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 in 181 msec 2024-11-14T07:17:25,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-14T07:17:25,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, REOPEN/MOVE in 511 msec 2024-11-14T07:17:25,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-14T07:17:25,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-14T07:17:25,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-11-14T07:17:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T07:17:25,279 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:25,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,280 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61191270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,282 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f644b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,284 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284d26cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,286 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d904c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,289 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129748ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,291 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661c27e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,293 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5028d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,295 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56d9257d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18987f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,299 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac42df2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,302 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7386c5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,303 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bdd349f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,306 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79df4039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20415df9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@186630c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,311 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260cd62e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625675f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299a1e83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:25,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:25,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108f5a74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:25,321 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcdbfce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:25,322 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,322 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,322 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,323 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,323 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:25,323 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,323 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,323 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,323 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T07:17:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,324 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,324 DEBUG [Thread-1483 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,324 DEBUG [Thread-1483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,324 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:25,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:17:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,326 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:25,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:25,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,327 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:25,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:17:25,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:17:25,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:25,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:25,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:25,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:25,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:25,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:25,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111434c81f0584a641fc9e56665b59293761_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568645334/Put/seqid=0 2024-11-14T07:17:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742146_1322 (size=14594) 2024-11-14T07:17:25,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568645391, max timestamp=1731568645403Num uniques 859; 2024-11-14T07:17:25,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568645391, max timestamp=1731568645403Num uniques 859; 2024-11-14T07:17:25,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731568645391, max timestamp=1731568645403Num uniques 859; 2024-11-14T07:17:25,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:17:25,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:17:25,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:17:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:17:25,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568705422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568705427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568705431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568705432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568705433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568705542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568705542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568705542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568705552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568705552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:17:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568705752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568705752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568705752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,768 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568705762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568705762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:25,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111434c81f0584a641fc9e56665b59293761_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111434c81f0584a641fc9e56665b59293761_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:25,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9e894732083241f7b93c42a884bc2c5c, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:25,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9e894732083241f7b93c42a884bc2c5c is 175, key is test_row_0/A:col10/1731568645334/Put/seqid=0 2024-11-14T07:17:25,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742147_1323 (size=39549) 2024-11-14T07:17:25,783 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9e894732083241f7b93c42a884bc2c5c 2024-11-14T07:17:25,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:25,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:25,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/9b9aea32f344499ca13956bafdb3d64e is 50, key is test_row_0/B:col10/1731568645334/Put/seqid=0 2024-11-14T07:17:25,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742148_1324 (size=12001) 2024-11-14T07:17:25,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:25,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:25,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:25,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:17:26,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568706061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568706071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568706072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568706082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568706082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:26,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:26,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/9b9aea32f344499ca13956bafdb3d64e 2024-11-14T07:17:26,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/45b5386f844d434ab3362575320f4620 is 50, key is test_row_0/C:col10/1731568645334/Put/seqid=0 2024-11-14T07:17:26,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742149_1325 (size=12001) 2024-11-14T07:17:26,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:26,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:26,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:26,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:26,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:17:26,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:26,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:26,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:26,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568706581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568706582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568706582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568706592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568706602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:26,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/45b5386f844d434ab3362575320f4620 2024-11-14T07:17:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9e894732083241f7b93c42a884bc2c5c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9e894732083241f7b93c42a884bc2c5c 2024-11-14T07:17:26,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9e894732083241f7b93c42a884bc2c5c, entries=200, sequenceid=15, filesize=38.6 K 2024-11-14T07:17:26,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/9b9aea32f344499ca13956bafdb3d64e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9b9aea32f344499ca13956bafdb3d64e 2024-11-14T07:17:26,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9b9aea32f344499ca13956bafdb3d64e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:17:26,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/45b5386f844d434ab3362575320f4620 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/45b5386f844d434ab3362575320f4620 2024-11-14T07:17:26,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/45b5386f844d434ab3362575320f4620, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:17:26,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 17b20244019df3f0f04e41107ed48efc in 1324ms, sequenceid=15, compaction requested=false 2024-11-14T07:17:26,661 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T07:17:26,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:26,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T07:17:26,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:26,705 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-14T07:17:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148d4655985f5944e18af310ee8c70b9e7_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568645429/Put/seqid=0 2024-11-14T07:17:26,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742150_1326 (size=12154) 2024-11-14T07:17:26,954 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:17:27,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:27,139 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148d4655985f5944e18af310ee8c70b9e7_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148d4655985f5944e18af310ee8c70b9e7_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:27,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/027ff4355c4346b28d4435f74a7ab431, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/027ff4355c4346b28d4435f74a7ab431 is 175, key is test_row_0/A:col10/1731568645429/Put/seqid=0 2024-11-14T07:17:27,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742151_1327 (size=30955) 2024-11-14T07:17:27,143 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/027ff4355c4346b28d4435f74a7ab431 2024-11-14T07:17:27,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/cd612fcec7a44fbfac5216d8f97ae207 is 50, key is test_row_0/B:col10/1731568645429/Put/seqid=0 2024-11-14T07:17:27,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742152_1328 (size=12001) 2024-11-14T07:17:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:17:27,553 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/cd612fcec7a44fbfac5216d8f97ae207 2024-11-14T07:17:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/29602c8c5ccc4104ac622d28536a0e63 is 50, key is test_row_0/C:col10/1731568645429/Put/seqid=0 2024-11-14T07:17:27,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742153_1329 (size=12001) 2024-11-14T07:17:27,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568707621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568707622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568707622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,627 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568707623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568707626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568707731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568707732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568707742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568707742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568707942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568707942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,967 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/29602c8c5ccc4104ac622d28536a0e63 2024-11-14T07:17:27,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568707962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568707962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:27,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/027ff4355c4346b28d4435f74a7ab431 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/027ff4355c4346b28d4435f74a7ab431 2024-11-14T07:17:27,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/027ff4355c4346b28d4435f74a7ab431, entries=150, sequenceid=49, filesize=30.2 K 2024-11-14T07:17:27,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/cd612fcec7a44fbfac5216d8f97ae207 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/cd612fcec7a44fbfac5216d8f97ae207 2024-11-14T07:17:27,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/cd612fcec7a44fbfac5216d8f97ae207, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T07:17:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/29602c8c5ccc4104ac622d28536a0e63 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/29602c8c5ccc4104ac622d28536a0e63 2024-11-14T07:17:28,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/29602c8c5ccc4104ac622d28536a0e63, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T07:17:28,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 17b20244019df3f0f04e41107ed48efc in 1297ms, sequenceid=49, compaction requested=false 2024-11-14T07:17:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T07:17:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T07:17:28,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T07:17:28,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6750 sec 2024-11-14T07:17:28,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.6860 sec 2024-11-14T07:17:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:28,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:17:28,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:28,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:28,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:28,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:28,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:28,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:28,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d99f8a3d4db341debe29f59bbbdba48a_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568647624/Put/seqid=0 2024-11-14T07:17:28,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742154_1330 (size=14594) 2024-11-14T07:17:28,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:28,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d99f8a3d4db341debe29f59bbbdba48a_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d99f8a3d4db341debe29f59bbbdba48a_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:28,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/77db160c597b486c8f0783ed7505663f, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:28,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/77db160c597b486c8f0783ed7505663f is 175, key is test_row_0/A:col10/1731568647624/Put/seqid=0 2024-11-14T07:17:28,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742155_1331 (size=39549) 2024-11-14T07:17:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568708315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568708316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568708327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568708328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568708431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568708432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568708443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568708451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568708642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568708642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568708661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568708672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,692 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/77db160c597b486c8f0783ed7505663f 2024-11-14T07:17:28,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/fc75405d2acb48358f127454b43a0e72 is 50, key is test_row_0/B:col10/1731568647624/Put/seqid=0 2024-11-14T07:17:28,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742156_1332 (size=12001) 2024-11-14T07:17:28,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/fc75405d2acb48358f127454b43a0e72 2024-11-14T07:17:28,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/2798a6d90ee741648145f500bc3795e4 is 50, key is test_row_0/C:col10/1731568647624/Put/seqid=0 2024-11-14T07:17:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742157_1333 (size=12001) 2024-11-14T07:17:28,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/2798a6d90ee741648145f500bc3795e4 2024-11-14T07:17:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/77db160c597b486c8f0783ed7505663f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77db160c597b486c8f0783ed7505663f 2024-11-14T07:17:28,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77db160c597b486c8f0783ed7505663f, entries=200, sequenceid=61, filesize=38.6 K 2024-11-14T07:17:28,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/fc75405d2acb48358f127454b43a0e72 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/fc75405d2acb48358f127454b43a0e72 2024-11-14T07:17:28,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/fc75405d2acb48358f127454b43a0e72, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T07:17:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/2798a6d90ee741648145f500bc3795e4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/2798a6d90ee741648145f500bc3795e4 2024-11-14T07:17:28,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/2798a6d90ee741648145f500bc3795e4, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T07:17:28,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 17b20244019df3f0f04e41107ed48efc in 517ms, sequenceid=61, compaction requested=true 2024-11-14T07:17:28,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:28,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:28,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:28,781 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:28,781 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:28,782 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:28,782 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:28,782 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/A is initiating minor compaction (all files) 2024-11-14T07:17:28,782 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/B is initiating minor compaction (all files) 2024-11-14T07:17:28,782 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/A in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:28,782 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/B in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:28,782 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9e894732083241f7b93c42a884bc2c5c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/027ff4355c4346b28d4435f74a7ab431, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77db160c597b486c8f0783ed7505663f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=107.5 K 2024-11-14T07:17:28,782 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9b9aea32f344499ca13956bafdb3d64e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/cd612fcec7a44fbfac5216d8f97ae207, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/fc75405d2acb48358f127454b43a0e72] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=35.2 K 2024-11-14T07:17:28,782 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:28,783 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9e894732083241f7b93c42a884bc2c5c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/027ff4355c4346b28d4435f74a7ab431, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77db160c597b486c8f0783ed7505663f] 2024-11-14T07:17:28,783 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b9aea32f344499ca13956bafdb3d64e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731568645331 2024-11-14T07:17:28,783 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e894732083241f7b93c42a884bc2c5c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731568645331 2024-11-14T07:17:28,783 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cd612fcec7a44fbfac5216d8f97ae207, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731568645409 2024-11-14T07:17:28,783 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027ff4355c4346b28d4435f74a7ab431, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731568645409 2024-11-14T07:17:28,783 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting fc75405d2acb48358f127454b43a0e72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731568647622 2024-11-14T07:17:28,783 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77db160c597b486c8f0783ed7505663f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731568647610 2024-11-14T07:17:28,791 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:28,793 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:28,793 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114769da516f95e41fb95b4b63d5357bb85_17b20244019df3f0f04e41107ed48efc store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:28,793 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/74a5d8ede6e84aeebd9e8f31ad800054 is 50, key is test_row_0/B:col10/1731568647624/Put/seqid=0 2024-11-14T07:17:28,796 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114769da516f95e41fb95b4b63d5357bb85_17b20244019df3f0f04e41107ed48efc, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:28,796 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114769da516f95e41fb95b4b63d5357bb85_17b20244019df3f0f04e41107ed48efc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:28,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742158_1334 (size=12104) 2024-11-14T07:17:28,808 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/74a5d8ede6e84aeebd9e8f31ad800054 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/74a5d8ede6e84aeebd9e8f31ad800054 2024-11-14T07:17:28,815 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/B of 17b20244019df3f0f04e41107ed48efc into 74a5d8ede6e84aeebd9e8f31ad800054(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:28,815 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:28,815 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/B, priority=13, startTime=1731568648781; duration=0sec 2024-11-14T07:17:28,815 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:28,815 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:28,815 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:28,816 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:28,816 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/C is initiating minor compaction (all files) 2024-11-14T07:17:28,816 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/C in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:28,817 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/45b5386f844d434ab3362575320f4620, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/29602c8c5ccc4104ac622d28536a0e63, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/2798a6d90ee741648145f500bc3795e4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=35.2 K 2024-11-14T07:17:28,817 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b5386f844d434ab3362575320f4620, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731568645331 2024-11-14T07:17:28,817 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 29602c8c5ccc4104ac622d28536a0e63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731568645409 2024-11-14T07:17:28,818 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2798a6d90ee741648145f500bc3795e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731568647622 2024-11-14T07:17:28,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742159_1335 (size=4469) 2024-11-14T07:17:28,825 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:28,825 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/bb82f657ace34b16989a9e5ea5c3781a is 50, key is test_row_0/C:col10/1731568647624/Put/seqid=0 2024-11-14T07:17:28,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742160_1336 (size=12104) 2024-11-14T07:17:28,840 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/bb82f657ace34b16989a9e5ea5c3781a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/bb82f657ace34b16989a9e5ea5c3781a 2024-11-14T07:17:28,847 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/C of 17b20244019df3f0f04e41107ed48efc into bb82f657ace34b16989a9e5ea5c3781a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:28,847 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:28,847 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/C, priority=13, startTime=1731568648782; duration=0sec 2024-11-14T07:17:28,848 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:28,848 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:28,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T07:17:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:28,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114391635d5af7b440381633c53087e4f74_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568648953/Put/seqid=0 2024-11-14T07:17:28,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742161_1337 (size=14594) 2024-11-14T07:17:28,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568708978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568708978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568708982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568708983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568709092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568709092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568709102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,221 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#A#compaction#282 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:29,221 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/479be62f89a244f38851d48d94765326 is 175, key is test_row_0/A:col10/1731568647624/Put/seqid=0 2024-11-14T07:17:29,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742162_1338 (size=31058) 2024-11-14T07:17:29,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568709302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568709302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568709311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:29,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114391635d5af7b440381633c53087e4f74_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114391635d5af7b440381633c53087e4f74_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:29,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/01ae4df667aa48d2b44fdf1f41febee7, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:29,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/01ae4df667aa48d2b44fdf1f41febee7 is 175, key is test_row_0/A:col10/1731568648953/Put/seqid=0 2024-11-14T07:17:29,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742163_1339 (size=39549) 2024-11-14T07:17:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:17:29,482 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:29,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T07:17:29,484 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T07:17:29,485 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:29,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:29,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568709502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T07:17:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568709611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,631 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/479be62f89a244f38851d48d94765326 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/479be62f89a244f38851d48d94765326 2024-11-14T07:17:29,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568709622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T07:17:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568709632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:29,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:29,643 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/A of 17b20244019df3f0f04e41107ed48efc into 479be62f89a244f38851d48d94765326(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:29,643 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:29,643 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/A, priority=13, startTime=1731568648781; duration=0sec 2024-11-14T07:17:29,644 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:29,644 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:29,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568709642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:29,656 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:29,782 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/01ae4df667aa48d2b44fdf1f41febee7 2024-11-14T07:17:29,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/9eaf8a76d524462681e615b3fc30fe04 is 50, key is test_row_0/B:col10/1731568648953/Put/seqid=0 2024-11-14T07:17:29,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T07:17:29,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:29,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:29,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:29,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:29,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T07:17:29,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742164_1340 (size=12001) 2024-11-14T07:17:29,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/9eaf8a76d524462681e615b3fc30fe04 2024-11-14T07:17:29,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f600b4740a96424aace592b08f251a9e is 50, key is test_row_0/C:col10/1731568648953/Put/seqid=0 2024-11-14T07:17:29,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742165_1341 (size=12001) 2024-11-14T07:17:29,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f600b4740a96424aace592b08f251a9e 2024-11-14T07:17:29,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/01ae4df667aa48d2b44fdf1f41febee7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/01ae4df667aa48d2b44fdf1f41febee7 2024-11-14T07:17:29,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/01ae4df667aa48d2b44fdf1f41febee7, entries=200, sequenceid=84, filesize=38.6 K 2024-11-14T07:17:29,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/9eaf8a76d524462681e615b3fc30fe04 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9eaf8a76d524462681e615b3fc30fe04 2024-11-14T07:17:29,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9eaf8a76d524462681e615b3fc30fe04, entries=150, sequenceid=84, filesize=11.7 K 2024-11-14T07:17:29,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f600b4740a96424aace592b08f251a9e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f600b4740a96424aace592b08f251a9e 2024-11-14T07:17:29,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f600b4740a96424aace592b08f251a9e, entries=150, sequenceid=84, filesize=11.7 K 2024-11-14T07:17:29,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 17b20244019df3f0f04e41107ed48efc in 881ms, sequenceid=84, compaction requested=false 2024-11-14T07:17:29,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:29,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T07:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:29,944 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:29,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:29,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114902498bf051c479d9885460cd77935a4_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568648981/Put/seqid=0 2024-11-14T07:17:29,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742166_1342 (size=12154) 2024-11-14T07:17:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T07:17:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:30,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:30,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568650170, max timestamp=1731568650195Num uniques 859; 2024-11-14T07:17:30,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568650170, max timestamp=1731568650195Num uniques 859; 2024-11-14T07:17:30,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:30,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:30,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568710216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568710223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568710224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568710332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568710332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568710342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:30,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114902498bf051c479d9885460cd77935a4_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114902498bf051c479d9885460cd77935a4_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:30,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a554e1c6783c4f1eba535fd984ce82d8, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:30,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a554e1c6783c4f1eba535fd984ce82d8 is 175, key is test_row_0/A:col10/1731568648981/Put/seqid=0 2024-11-14T07:17:30,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742167_1343 (size=30955) 2024-11-14T07:17:30,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568710522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,527 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568710551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568710552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568710562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T07:17:30,771 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a554e1c6783c4f1eba535fd984ce82d8 2024-11-14T07:17:30,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ca5d32e344e04171a98f3b8330a5b771 is 50, key is test_row_0/B:col10/1731568648981/Put/seqid=0 2024-11-14T07:17:30,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742168_1344 (size=12001) 2024-11-14T07:17:30,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568710862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568710871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568710872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:31,181 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ca5d32e344e04171a98f3b8330a5b771 2024-11-14T07:17:31,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/46af8dd41b9a48418781bb2693aa466c is 50, key is test_row_0/C:col10/1731568648981/Put/seqid=0 2024-11-14T07:17:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742169_1345 (size=12001) 2024-11-14T07:17:31,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568711372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:31,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568711392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:31,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568711392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:31,590 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/46af8dd41b9a48418781bb2693aa466c 2024-11-14T07:17:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a554e1c6783c4f1eba535fd984ce82d8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a554e1c6783c4f1eba535fd984ce82d8 2024-11-14T07:17:31,600 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a554e1c6783c4f1eba535fd984ce82d8, entries=150, sequenceid=96, filesize=30.2 K 2024-11-14T07:17:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ca5d32e344e04171a98f3b8330a5b771 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ca5d32e344e04171a98f3b8330a5b771 2024-11-14T07:17:31,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ca5d32e344e04171a98f3b8330a5b771, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T07:17:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/46af8dd41b9a48418781bb2693aa466c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46af8dd41b9a48418781bb2693aa466c 2024-11-14T07:17:31,615 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46af8dd41b9a48418781bb2693aa466c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T07:17:31,616 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 17b20244019df3f0f04e41107ed48efc in 1672ms, sequenceid=96, compaction requested=true 2024-11-14T07:17:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T07:17:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T07:17:31,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T07:17:31,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1330 sec 2024-11-14T07:17:31,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.1390 sec 2024-11-14T07:17:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T07:17:31,634 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:31,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T07:17:31,638 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:31,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:31,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T07:17:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T07:17:31,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T07:17:31,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:31,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=123.00 KB heapSize=353.63 KB 2024-11-14T07:17:31,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111404791f511a3342549db61ae181f174b2_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568650222/Put/seqid=0 2024-11-14T07:17:31,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742170_1346 (size=12154) 2024-11-14T07:17:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T07:17:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:32,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111404791f511a3342549db61ae181f174b2_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111404791f511a3342549db61ae181f174b2_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/bbef56b9344f40caa40f41400eaf2673, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:32,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/bbef56b9344f40caa40f41400eaf2673 is 175, key is test_row_0/A:col10/1731568650222/Put/seqid=0 2024-11-14T07:17:32,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742171_1347 (size=30955) 2024-11-14T07:17:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T07:17:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:32,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:32,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568712438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568712438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568712439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568712542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,548 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568712561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568712561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568712561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,625 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=126, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/bbef56b9344f40caa40f41400eaf2673 2024-11-14T07:17:32,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/f56d1a63f8e14d758859ec309ffeef20 is 50, key is test_row_0/B:col10/1731568650222/Put/seqid=0 2024-11-14T07:17:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742172_1348 (size=12001) 2024-11-14T07:17:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T07:17:32,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568712772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568712772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568712772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,039 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/f56d1a63f8e14d758859ec309ffeef20 2024-11-14T07:17:33,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f is 50, key is test_row_0/C:col10/1731568650222/Put/seqid=0 2024-11-14T07:17:33,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742173_1349 (size=12001) 2024-11-14T07:17:33,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f 2024-11-14T07:17:33,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/bbef56b9344f40caa40f41400eaf2673 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/bbef56b9344f40caa40f41400eaf2673 2024-11-14T07:17:33,072 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/bbef56b9344f40caa40f41400eaf2673, entries=150, sequenceid=126, filesize=30.2 K 2024-11-14T07:17:33,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/f56d1a63f8e14d758859ec309ffeef20 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f56d1a63f8e14d758859ec309ffeef20 2024-11-14T07:17:33,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f56d1a63f8e14d758859ec309ffeef20, entries=150, sequenceid=126, filesize=11.7 K 2024-11-14T07:17:33,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f 2024-11-14T07:17:33,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f, entries=150, sequenceid=126, filesize=11.7 K 2024-11-14T07:17:33,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~353.58 KB/362064, currentSize=53.67 KB/54960 for 17b20244019df3f0f04e41107ed48efc in 1291ms, sequenceid=126, compaction requested=true 2024-11-14T07:17:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T07:17:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T07:17:33,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T07:17:33,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-14T07:17:33,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4530 sec 2024-11-14T07:17:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:33,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:17:33,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:33,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:33,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:33,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:33,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:33,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:33,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142583a76e50d0495685f1f3fa4e0dccd4_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568652418/Put/seqid=0 2024-11-14T07:17:33,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742174_1350 (size=14794) 2024-11-14T07:17:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568713165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731568653140, max timestamp=1731568653164Num uniques 859; 2024-11-14T07:17:33,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731568653140, max timestamp=1731568653164Num uniques 859; 2024-11-14T07:17:33,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:17:33,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T07:17:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731568653144, max timestamp=1731568653164Num uniques 859; 2024-11-14T07:17:33,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:33,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568713247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568713255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568713272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568713372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568713372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568713492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:33,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142583a76e50d0495685f1f3fa4e0dccd4_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142583a76e50d0495685f1f3fa4e0dccd4_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:33,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/7e0cf78d8e864cb592153bb8eea3700a, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:33,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/7e0cf78d8e864cb592153bb8eea3700a is 175, key is test_row_0/A:col10/1731568652418/Put/seqid=0 2024-11-14T07:17:33,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742175_1351 (size=39749) 2024-11-14T07:17:33,549 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/7e0cf78d8e864cb592153bb8eea3700a 2024-11-14T07:17:33,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d2b914c40d9647149327c8dc252b439a is 50, key is test_row_0/B:col10/1731568652418/Put/seqid=0 2024-11-14T07:17:33,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568713596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568713596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742176_1352 (size=12151) 2024-11-14T07:17:33,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d2b914c40d9647149327c8dc252b439a 2024-11-14T07:17:33,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/3e2ac6f4128d4a58853a4e9238f31e92 is 50, key is test_row_0/C:col10/1731568652418/Put/seqid=0 2024-11-14T07:17:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742177_1353 (size=12151) 2024-11-14T07:17:33,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/3e2ac6f4128d4a58853a4e9238f31e92 2024-11-14T07:17:33,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/7e0cf78d8e864cb592153bb8eea3700a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/7e0cf78d8e864cb592153bb8eea3700a 2024-11-14T07:17:33,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/7e0cf78d8e864cb592153bb8eea3700a, entries=200, sequenceid=138, filesize=38.8 K 2024-11-14T07:17:33,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568713683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,689 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d2b914c40d9647149327c8dc252b439a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d2b914c40d9647149327c8dc252b439a 2024-11-14T07:17:33,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d2b914c40d9647149327c8dc252b439a, entries=150, sequenceid=138, filesize=11.9 K 2024-11-14T07:17:33,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/3e2ac6f4128d4a58853a4e9238f31e92 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/3e2ac6f4128d4a58853a4e9238f31e92 2024-11-14T07:17:33,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/3e2ac6f4128d4a58853a4e9238f31e92, entries=150, sequenceid=138, filesize=11.9 K 2024-11-14T07:17:33,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=125.23 KB/128240 for 17b20244019df3f0f04e41107ed48efc in 601ms, sequenceid=138, compaction requested=true 2024-11-14T07:17:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:33,703 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:17:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:17:33,703 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:17:33,707 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172266 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:17:33,707 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60258 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:17:33,707 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/A is initiating minor compaction (all files) 2024-11-14T07:17:33,707 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/B is initiating minor compaction (all files) 2024-11-14T07:17:33,707 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/A in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:33,707 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/B in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:33,708 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/479be62f89a244f38851d48d94765326, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/01ae4df667aa48d2b44fdf1f41febee7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a554e1c6783c4f1eba535fd984ce82d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/bbef56b9344f40caa40f41400eaf2673, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/7e0cf78d8e864cb592153bb8eea3700a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=168.2 K 2024-11-14T07:17:33,708 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/74a5d8ede6e84aeebd9e8f31ad800054, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9eaf8a76d524462681e615b3fc30fe04, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ca5d32e344e04171a98f3b8330a5b771, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f56d1a63f8e14d758859ec309ffeef20, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d2b914c40d9647149327c8dc252b439a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=58.8 K 2024-11-14T07:17:33,708 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:33,708 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/479be62f89a244f38851d48d94765326, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/01ae4df667aa48d2b44fdf1f41febee7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a554e1c6783c4f1eba535fd984ce82d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/bbef56b9344f40caa40f41400eaf2673, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/7e0cf78d8e864cb592153bb8eea3700a] 2024-11-14T07:17:33,708 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a5d8ede6e84aeebd9e8f31ad800054, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731568647622 2024-11-14T07:17:33,708 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479be62f89a244f38851d48d94765326, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731568647622 2024-11-14T07:17:33,708 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eaf8a76d524462681e615b3fc30fe04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731568648313 2024-11-14T07:17:33,709 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01ae4df667aa48d2b44fdf1f41febee7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731568648313 2024-11-14T07:17:33,709 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5d32e344e04171a98f3b8330a5b771, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568648969 2024-11-14T07:17:33,709 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a554e1c6783c4f1eba535fd984ce82d8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568648969 2024-11-14T07:17:33,709 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f56d1a63f8e14d758859ec309ffeef20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731568650186 2024-11-14T07:17:33,709 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b914c40d9647149327c8dc252b439a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731568652418 2024-11-14T07:17:33,710 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbef56b9344f40caa40f41400eaf2673, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731568650186 2024-11-14T07:17:33,710 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0cf78d8e864cb592153bb8eea3700a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731568652418 2024-11-14T07:17:33,729 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#B#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:33,730 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/0ae35b3bcb17403b97d903b77bfbddb3 is 50, key is test_row_0/B:col10/1731568652418/Put/seqid=0 2024-11-14T07:17:33,736 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:33,746 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f05d6e8753e243d88bf4f1d06a74683f_17b20244019df3f0f04e41107ed48efc store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:33,750 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f05d6e8753e243d88bf4f1d06a74683f_17b20244019df3f0f04e41107ed48efc, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:33,750 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f05d6e8753e243d88bf4f1d06a74683f_17b20244019df3f0f04e41107ed48efc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:33,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742178_1354 (size=12425) 2024-11-14T07:17:33,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742179_1355 (size=4469) 2024-11-14T07:17:33,778 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#A#compaction#298 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:33,779 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a3cbbe627cdb440ba142a84efa08a009 is 175, key is test_row_0/A:col10/1731568652418/Put/seqid=0 2024-11-14T07:17:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T07:17:33,782 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:33,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T07:17:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T07:17:33,785 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:33,786 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:33,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:33,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=131.94 KB heapSize=370.53 KB 2024-11-14T07:17:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:33,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:33,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742180_1356 (size=31379) 2024-11-14T07:17:33,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486a00c4ba0c845028aa1cad0823a8025_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568653253/Put/seqid=0 2024-11-14T07:17:33,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742181_1357 (size=14794) 2024-11-14T07:17:33,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:33,854 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486a00c4ba0c845028aa1cad0823a8025_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486a00c4ba0c845028aa1cad0823a8025_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:33,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/24a50aae1f9440d5b3e90fbd6c7bd689, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:33,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/24a50aae1f9440d5b3e90fbd6c7bd689 is 175, key is test_row_0/A:col10/1731568653253/Put/seqid=0 2024-11-14T07:17:33,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742182_1358 (size=39749) 2024-11-14T07:17:33,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/24a50aae1f9440d5b3e90fbd6c7bd689 2024-11-14T07:17:33,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/2eaea9ae3fc34e478941985ee7cb5c83 is 50, key is test_row_0/B:col10/1731568653253/Put/seqid=0 2024-11-14T07:17:33,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742183_1359 (size=12151) 2024-11-14T07:17:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T07:17:33,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568713922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568713922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:33,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T07:17:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568713926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:33,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:33,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:33,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:33,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568714042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:34,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T07:17:34,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:34,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T07:17:34,163 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/0ae35b3bcb17403b97d903b77bfbddb3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ae35b3bcb17403b97d903b77bfbddb3 2024-11-14T07:17:34,174 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/B of 17b20244019df3f0f04e41107ed48efc into 0ae35b3bcb17403b97d903b77bfbddb3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:34,175 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:34,175 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/B, priority=11, startTime=1731568653703; duration=0sec 2024-11-14T07:17:34,175 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:34,175 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:34,175 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:17:34,177 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60258 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:17:34,177 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/C is initiating minor compaction (all files) 2024-11-14T07:17:34,180 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/C in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,180 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/bb82f657ace34b16989a9e5ea5c3781a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f600b4740a96424aace592b08f251a9e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46af8dd41b9a48418781bb2693aa466c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/3e2ac6f4128d4a58853a4e9238f31e92] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=58.8 K 2024-11-14T07:17:34,181 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting bb82f657ace34b16989a9e5ea5c3781a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731568647622 2024-11-14T07:17:34,181 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f600b4740a96424aace592b08f251a9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731568648313 2024-11-14T07:17:34,182 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 46af8dd41b9a48418781bb2693aa466c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568648969 2024-11-14T07:17:34,182 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting b8b4ec0b8b2d41e6b2de3aacc7d5ae7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731568650186 2024-11-14T07:17:34,182 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e2ac6f4128d4a58853a4e9238f31e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731568652418 2024-11-14T07:17:34,203 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#C#compaction#301 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:34,203 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/90f1a33f416a4c04be3ae1de5b82a912 is 50, key is test_row_0/C:col10/1731568652418/Put/seqid=0 2024-11-14T07:17:34,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742184_1360 (size=12425) 2024-11-14T07:17:34,225 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/90f1a33f416a4c04be3ae1de5b82a912 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/90f1a33f416a4c04be3ae1de5b82a912 2024-11-14T07:17:34,234 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/C of 17b20244019df3f0f04e41107ed48efc into 90f1a33f416a4c04be3ae1de5b82a912(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:34,234 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:34,234 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/C, priority=11, startTime=1731568653703; duration=0sec 2024-11-14T07:17:34,234 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:34,234 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:34,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T07:17:34,245 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a3cbbe627cdb440ba142a84efa08a009 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a3cbbe627cdb440ba142a84efa08a009 2024-11-14T07:17:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,251 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/A of 17b20244019df3f0f04e41107ed48efc into a3cbbe627cdb440ba142a84efa08a009(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:34,251 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:34,251 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/A, priority=11, startTime=1731568653703; duration=0sec 2024-11-14T07:17:34,251 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:34,251 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:34,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568714262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:34,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/2eaea9ae3fc34e478941985ee7cb5c83 2024-11-14T07:17:34,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/e70a701faac14695ace397952284265c is 50, key is test_row_0/C:col10/1731568653253/Put/seqid=0 2024-11-14T07:17:34,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742185_1361 (size=12151) 2024-11-14T07:17:34,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T07:17:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T07:17:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568714442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568714442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:34,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T07:17:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568714582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:34,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T07:17:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:34,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/e70a701faac14695ace397952284265c 2024-11-14T07:17:34,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/24a50aae1f9440d5b3e90fbd6c7bd689 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/24a50aae1f9440d5b3e90fbd6c7bd689 2024-11-14T07:17:34,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/24a50aae1f9440d5b3e90fbd6c7bd689, entries=200, sequenceid=173, filesize=38.8 K 2024-11-14T07:17:34,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/2eaea9ae3fc34e478941985ee7cb5c83 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/2eaea9ae3fc34e478941985ee7cb5c83 2024-11-14T07:17:34,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/2eaea9ae3fc34e478941985ee7cb5c83, entries=150, sequenceid=173, filesize=11.9 K 2024-11-14T07:17:34,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/e70a701faac14695ace397952284265c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e70a701faac14695ace397952284265c 2024-11-14T07:17:34,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e70a701faac14695ace397952284265c, entries=150, sequenceid=173, filesize=11.9 K 2024-11-14T07:17:34,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~370.48 KB/379376, currentSize=53.67 KB/54960 for 17b20244019df3f0f04e41107ed48efc in 907ms, sequenceid=173, compaction requested=false 2024-11-14T07:17:34,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:34,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T07:17:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:34,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:17:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ae781d7dd6ed42a0b133e0cf90bb8bea_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568653899/Put/seqid=0 2024-11-14T07:17:34,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742186_1362 (size=12304) 2024-11-14T07:17:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T07:17:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:35,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568655180, max timestamp=1731568655210Num uniques 859; 2024-11-14T07:17:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568655180, max timestamp=1731568655210Num uniques 859; 2024-11-14T07:17:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568655180, max timestamp=1731568655210Num uniques 859; 2024-11-14T07:17:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:35,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:35,270 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ae781d7dd6ed42a0b133e0cf90bb8bea_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae781d7dd6ed42a0b133e0cf90bb8bea_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/6de37254321348d8bb6f62606cf0be70, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/6de37254321348d8bb6f62606cf0be70 is 175, key is test_row_0/A:col10/1731568653899/Put/seqid=0 2024-11-14T07:17:35,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742187_1363 (size=31105) 2024-11-14T07:17:35,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568715318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:35,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568715432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:35,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568715462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:35,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568715462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:35,469 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:35,469 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:35,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568715642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:35,675 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/6de37254321348d8bb6f62606cf0be70 2024-11-14T07:17:35,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ab252864b2c04276af660ec77c7e1a2f is 50, key is test_row_0/B:col10/1731568653899/Put/seqid=0 2024-11-14T07:17:35,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742188_1364 (size=12151) 2024-11-14T07:17:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T07:17:35,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568715961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:36,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ab252864b2c04276af660ec77c7e1a2f 2024-11-14T07:17:36,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/ca6f7bc1793848e9b4c36e431544c596 is 50, key is test_row_0/C:col10/1731568653899/Put/seqid=0 2024-11-14T07:17:36,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742189_1365 (size=12151) 2024-11-14T07:17:36,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568716472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:36,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:36,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:36,498 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/ca6f7bc1793848e9b4c36e431544c596 2024-11-14T07:17:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/6de37254321348d8bb6f62606cf0be70 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/6de37254321348d8bb6f62606cf0be70 2024-11-14T07:17:36,507 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/6de37254321348d8bb6f62606cf0be70, entries=150, sequenceid=187, filesize=30.4 K 2024-11-14T07:17:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ab252864b2c04276af660ec77c7e1a2f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ab252864b2c04276af660ec77c7e1a2f 2024-11-14T07:17:36,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ab252864b2c04276af660ec77c7e1a2f, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T07:17:36,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/ca6f7bc1793848e9b4c36e431544c596 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/ca6f7bc1793848e9b4c36e431544c596 2024-11-14T07:17:36,516 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/ca6f7bc1793848e9b4c36e431544c596, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T07:17:36,516 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 17b20244019df3f0f04e41107ed48efc in 1660ms, sequenceid=187, compaction requested=true 2024-11-14T07:17:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T07:17:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T07:17:36,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T07:17:36,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7320 sec 2024-11-14T07:17:36,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.7370 sec 2024-11-14T07:17:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:36,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T07:17:36,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:36,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:36,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:36,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:36,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:36,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:36,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114586bc08b48904941b83c00a3bed2c501_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568655303/Put/seqid=0 2024-11-14T07:17:36,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742190_1366 (size=14794) 2024-11-14T07:17:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568716677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:36,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568716781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:36,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:36,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:36,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114586bc08b48904941b83c00a3bed2c501_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114586bc08b48904941b83c00a3bed2c501_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:37,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/febb1fa9106948fa8e062c9d4747d21a, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:37,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/febb1fa9106948fa8e062c9d4747d21a is 175, key is test_row_0/A:col10/1731568655303/Put/seqid=0 2024-11-14T07:17:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568717002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742191_1367 (size=39749) 2024-11-14T07:17:37,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/febb1fa9106948fa8e062c9d4747d21a 2024-11-14T07:17:37,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/374c152a1d174f8fa2374d915d15e0cc is 50, key is test_row_0/B:col10/1731568655303/Put/seqid=0 2024-11-14T07:17:37,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742192_1368 (size=12151) 2024-11-14T07:17:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568717322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/374c152a1d174f8fa2374d915d15e0cc 2024-11-14T07:17:37,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/69cff8ecd4d848e1992a5102eee1625c is 50, key is test_row_0/C:col10/1731568655303/Put/seqid=0 2024-11-14T07:17:37,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742193_1369 (size=12151) 2024-11-14T07:17:37,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/69cff8ecd4d848e1992a5102eee1625c 2024-11-14T07:17:37,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/febb1fa9106948fa8e062c9d4747d21a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/febb1fa9106948fa8e062c9d4747d21a 2024-11-14T07:17:37,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/febb1fa9106948fa8e062c9d4747d21a, entries=200, sequenceid=223, filesize=38.8 K 2024-11-14T07:17:37,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/374c152a1d174f8fa2374d915d15e0cc as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/374c152a1d174f8fa2374d915d15e0cc 2024-11-14T07:17:37,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/374c152a1d174f8fa2374d915d15e0cc, entries=150, sequenceid=223, filesize=11.9 K 2024-11-14T07:17:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/69cff8ecd4d848e1992a5102eee1625c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/69cff8ecd4d848e1992a5102eee1625c 2024-11-14T07:17:37,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/69cff8ecd4d848e1992a5102eee1625c, entries=150, sequenceid=223, filesize=11.9 K 2024-11-14T07:17:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 17b20244019df3f0f04e41107ed48efc in 863ms, sequenceid=223, compaction requested=true 2024-11-14T07:17:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:37,454 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:17:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:37,454 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:17:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,455 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141982 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:17:37,455 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:17:37,455 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/B is initiating minor compaction (all files) 2024-11-14T07:17:37,455 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/A is initiating minor compaction (all files) 2024-11-14T07:17:37,455 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/B in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:37,455 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/A in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:37,455 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ae35b3bcb17403b97d903b77bfbddb3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/2eaea9ae3fc34e478941985ee7cb5c83, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ab252864b2c04276af660ec77c7e1a2f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/374c152a1d174f8fa2374d915d15e0cc] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=47.7 K 2024-11-14T07:17:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,455 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a3cbbe627cdb440ba142a84efa08a009, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/24a50aae1f9440d5b3e90fbd6c7bd689, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/6de37254321348d8bb6f62606cf0be70, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/febb1fa9106948fa8e062c9d4747d21a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=138.7 K 2024-11-14T07:17:37,456 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:37,456 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a3cbbe627cdb440ba142a84efa08a009, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/24a50aae1f9440d5b3e90fbd6c7bd689, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/6de37254321348d8bb6f62606cf0be70, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/febb1fa9106948fa8e062c9d4747d21a] 2024-11-14T07:17:37,456 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae35b3bcb17403b97d903b77bfbddb3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731568652418 2024-11-14T07:17:37,456 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3cbbe627cdb440ba142a84efa08a009, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731568652418 2024-11-14T07:17:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,456 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eaea9ae3fc34e478941985ee7cb5c83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731568653227 2024-11-14T07:17:37,456 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24a50aae1f9440d5b3e90fbd6c7bd689, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731568653227 2024-11-14T07:17:37,456 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ab252864b2c04276af660ec77c7e1a2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731568653876 2024-11-14T07:17:37,456 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de37254321348d8bb6f62606cf0be70, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731568653876 2024-11-14T07:17:37,457 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 374c152a1d174f8fa2374d915d15e0cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731568655286 2024-11-14T07:17:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,457 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting febb1fa9106948fa8e062c9d4747d21a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731568655286 2024-11-14T07:17:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,463 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,467 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,467 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/a0f8266a85894d5a96de5ef221deb968 is 50, key is test_row_0/B:col10/1731568655303/Put/seqid=0 2024-11-14T07:17:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,472 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111494203419df0d4d0cbab63be679d4bc70_17b20244019df3f0f04e41107ed48efc store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,474 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111494203419df0d4d0cbab63be679d4bc70_17b20244019df3f0f04e41107ed48efc, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,474 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111494203419df0d4d0cbab63be679d4bc70_17b20244019df3f0f04e41107ed48efc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742194_1370 (size=12561) 2024-11-14T07:17:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742195_1371 (size=4469) 2024-11-14T07:17:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:17:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111435a91c936a804024935888fc74e2b46b_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568656675/Put/seqid=0 2024-11-14T07:17:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742196_1372 (size=12304) 2024-11-14T07:17:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,536 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,541 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111435a91c936a804024935888fc74e2b46b_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111435a91c936a804024935888fc74e2b46b_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/adb59187890b48e4becb38a14d5f1e49, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/adb59187890b48e4becb38a14d5f1e49 is 175, key is test_row_0/A:col10/1731568656675/Put/seqid=0 2024-11-14T07:17:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742197_1373 (size=31101) 2024-11-14T07:17:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,547 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/adb59187890b48e4becb38a14d5f1e49 2024-11-14T07:17:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/8d21cfa824844babbba9392540f46d38 is 50, key is test_row_0/B:col10/1731568656675/Put/seqid=0 2024-11-14T07:17:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742198_1374 (size=9757) 2024-11-14T07:17:37,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/8d21cfa824844babbba9392540f46d38 2024-11-14T07:17:37,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/d40e2bc087b943dbb744439c29aa30c7 is 50, key is test_row_0/C:col10/1731568656675/Put/seqid=0 2024-11-14T07:17:37,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742199_1375 (size=9757) 2024-11-14T07:17:37,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/d40e2bc087b943dbb744439c29aa30c7 2024-11-14T07:17:37,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/adb59187890b48e4becb38a14d5f1e49 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/adb59187890b48e4becb38a14d5f1e49 2024-11-14T07:17:37,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/adb59187890b48e4becb38a14d5f1e49, entries=150, sequenceid=234, filesize=30.4 K 2024-11-14T07:17:37,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/8d21cfa824844babbba9392540f46d38 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/8d21cfa824844babbba9392540f46d38 2024-11-14T07:17:37,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/8d21cfa824844babbba9392540f46d38, entries=100, sequenceid=234, filesize=9.5 K 2024-11-14T07:17:37,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/d40e2bc087b943dbb744439c29aa30c7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/d40e2bc087b943dbb744439c29aa30c7 2024-11-14T07:17:37,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/d40e2bc087b943dbb744439c29aa30c7, entries=100, sequenceid=234, filesize=9.5 K 2024-11-14T07:17:37,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for 17b20244019df3f0f04e41107ed48efc in 90ms, sequenceid=234, compaction requested=true 2024-11-14T07:17:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:17:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T07:17:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T07:17:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:37,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:17:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:37,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114549e5a8f6d5a456f8fb2fb0e41e35fb5_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568657581/Put/seqid=0 2024-11-14T07:17:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742200_1376 (size=14794) 2024-11-14T07:17:37,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568717653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568717654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568717658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568717772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568717772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568717772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568717841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,887 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:37,887 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/2888ded141d240ee97210d041b9ff420 is 175, key is test_row_0/A:col10/1731568655303/Put/seqid=0 2024-11-14T07:17:37,888 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/a0f8266a85894d5a96de5ef221deb968 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/a0f8266a85894d5a96de5ef221deb968 2024-11-14T07:17:37,893 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/B of 17b20244019df3f0f04e41107ed48efc into a0f8266a85894d5a96de5ef221deb968(size=12.3 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:37,893 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:37,893 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/B, priority=12, startTime=1731568657454; duration=0sec 2024-11-14T07:17:37,893 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T07:17:37,893 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:37,893 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-14T07:17:37,894 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:17:37,894 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:17:37,894 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. because compaction request was cancelled 2024-11-14T07:17:37,894 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:37,894 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:17:37,895 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58635 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:17:37,895 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/C is initiating minor compaction (all files) 2024-11-14T07:17:37,895 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/C in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:37,895 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/90f1a33f416a4c04be3ae1de5b82a912, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e70a701faac14695ace397952284265c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/ca6f7bc1793848e9b4c36e431544c596, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/69cff8ecd4d848e1992a5102eee1625c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/d40e2bc087b943dbb744439c29aa30c7] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=57.3 K 2024-11-14T07:17:37,896 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f1a33f416a4c04be3ae1de5b82a912, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731568652418 2024-11-14T07:17:37,896 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e70a701faac14695ace397952284265c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731568653227 2024-11-14T07:17:37,896 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ca6f7bc1793848e9b4c36e431544c596, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731568653876 2024-11-14T07:17:37,897 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 69cff8ecd4d848e1992a5102eee1625c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731568655286 2024-11-14T07:17:37,899 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d40e2bc087b943dbb744439c29aa30c7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731568656675 2024-11-14T07:17:37,912 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:37,913 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/14f54455ab6e4a8185262ded7804f7a2 is 50, key is test_row_0/C:col10/1731568656675/Put/seqid=0 2024-11-14T07:17:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742201_1377 (size=31515) 2024-11-14T07:17:37,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742202_1378 (size=12595) 2024-11-14T07:17:37,930 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/2888ded141d240ee97210d041b9ff420 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/2888ded141d240ee97210d041b9ff420 2024-11-14T07:17:37,939 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/A of 17b20244019df3f0f04e41107ed48efc into 2888ded141d240ee97210d041b9ff420(size=30.8 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:37,939 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:37,939 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/A, priority=12, startTime=1731568657454; duration=0sec 2024-11-14T07:17:37,939 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. because compaction request was cancelled 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:37,940 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T07:17:37,941 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:17:37,941 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:17:37,941 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. because compaction request was cancelled 2024-11-14T07:17:37,941 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T07:17:37,943 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:37,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T07:17:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:17:37,946 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:37,947 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:37,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:37,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568717992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568717992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568717992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,040 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:38,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114549e5a8f6d5a456f8fb2fb0e41e35fb5_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114549e5a8f6d5a456f8fb2fb0e41e35fb5_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:38,043 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/de4b9f530688434a9ec534906a6fd17c, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:38,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/de4b9f530688434a9ec534906a6fd17c is 175, key is test_row_0/A:col10/1731568657581/Put/seqid=0 2024-11-14T07:17:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742203_1379 (size=39749) 2024-11-14T07:17:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:17:38,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:17:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:17:38,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:38,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:17:38,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568718301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568718302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568718302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,330 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/14f54455ab6e4a8185262ded7804f7a2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/14f54455ab6e4a8185262ded7804f7a2 2024-11-14T07:17:38,334 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/C of 17b20244019df3f0f04e41107ed48efc into 14f54455ab6e4a8185262ded7804f7a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:38,334 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:38,334 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/C, priority=11, startTime=1731568657603; duration=0sec 2024-11-14T07:17:38,334 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:38,334 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:38,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:17:38,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:38,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:38,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/de4b9f530688434a9ec534906a6fd17c 2024-11-14T07:17:38,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d16e24e36bab423dbea55cd16deabd83 is 50, key is test_row_0/B:col10/1731568657581/Put/seqid=0 2024-11-14T07:17:38,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742204_1380 (size=12151) 2024-11-14T07:17:38,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d16e24e36bab423dbea55cd16deabd83 2024-11-14T07:17:38,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/0f5b428681384ab4ba160305361ec455 is 50, key is test_row_0/C:col10/1731568657581/Put/seqid=0 2024-11-14T07:17:38,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742205_1381 (size=12151) 2024-11-14T07:17:38,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/0f5b428681384ab4ba160305361ec455 2024-11-14T07:17:38,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/de4b9f530688434a9ec534906a6fd17c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/de4b9f530688434a9ec534906a6fd17c 2024-11-14T07:17:38,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/de4b9f530688434a9ec534906a6fd17c, entries=200, sequenceid=254, filesize=38.8 K 2024-11-14T07:17:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d16e24e36bab423dbea55cd16deabd83 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d16e24e36bab423dbea55cd16deabd83 2024-11-14T07:17:38,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d16e24e36bab423dbea55cd16deabd83, entries=150, sequenceid=254, filesize=11.9 K 2024-11-14T07:17:38,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/0f5b428681384ab4ba160305361ec455 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/0f5b428681384ab4ba160305361ec455 2024-11-14T07:17:38,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/0f5b428681384ab4ba160305361ec455, entries=150, sequenceid=254, filesize=11.9 K 2024-11-14T07:17:38,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 17b20244019df3f0f04e41107ed48efc in 875ms, sequenceid=254, compaction requested=true 2024-11-14T07:17:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:38,492 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:38,492 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/A is initiating minor compaction (all files) 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/B is initiating minor compaction (all files) 2024-11-14T07:17:38,493 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/B in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,493 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/A in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,493 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/2888ded141d240ee97210d041b9ff420, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/adb59187890b48e4becb38a14d5f1e49, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/de4b9f530688434a9ec534906a6fd17c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=100.0 K 2024-11-14T07:17:38,493 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/a0f8266a85894d5a96de5ef221deb968, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/8d21cfa824844babbba9392540f46d38, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d16e24e36bab423dbea55cd16deabd83] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=33.7 K 2024-11-14T07:17:38,493 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/2888ded141d240ee97210d041b9ff420, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/adb59187890b48e4becb38a14d5f1e49, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/de4b9f530688434a9ec534906a6fd17c] 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2888ded141d240ee97210d041b9ff420, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731568655286 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f8266a85894d5a96de5ef221deb968, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731568655286 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb59187890b48e4becb38a14d5f1e49, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731568656663 2024-11-14T07:17:38,493 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d21cfa824844babbba9392540f46d38, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731568656675 2024-11-14T07:17:38,494 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d16e24e36bab423dbea55cd16deabd83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731568657581 2024-11-14T07:17:38,494 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting de4b9f530688434a9ec534906a6fd17c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731568657581 2024-11-14T07:17:38,500 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:38,507 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411146ea9aaf161a54d6a9b6578ef140496cc_17b20244019df3f0f04e41107ed48efc store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:38,508 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:38,509 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d3a79775127e45debe62952d1442476b is 50, key is test_row_0/B:col10/1731568657581/Put/seqid=0 2024-11-14T07:17:38,509 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411146ea9aaf161a54d6a9b6578ef140496cc_17b20244019df3f0f04e41107ed48efc, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:38,509 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146ea9aaf161a54d6a9b6578ef140496cc_17b20244019df3f0f04e41107ed48efc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:38,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742206_1382 (size=4469) 2024-11-14T07:17:38,513 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#A#compaction#318 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:38,514 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/77fb254c479a4ec980b3068634515720 is 175, key is test_row_0/A:col10/1731568657581/Put/seqid=0 2024-11-14T07:17:38,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742207_1383 (size=12663) 2024-11-14T07:17:38,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742208_1384 (size=31617) 2024-11-14T07:17:38,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:17:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:38,559 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:17:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:17:38,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ceb6dd3f86ee4fe59b406e247832c349_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568657656/Put/seqid=0 2024-11-14T07:17:38,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742209_1385 (size=12454) 2024-11-14T07:17:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568718853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568718853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568718854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568718872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,879 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,933 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/d3a79775127e45debe62952d1442476b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d3a79775127e45debe62952d1442476b 2024-11-14T07:17:38,945 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/B of 17b20244019df3f0f04e41107ed48efc into d3a79775127e45debe62952d1442476b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:38,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:38,945 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/B, priority=13, startTime=1731568658492; duration=0sec 2024-11-14T07:17:38,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:38,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:38,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T07:17:38,947 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:17:38,947 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:17:38,948 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. because compaction request was cancelled 2024-11-14T07:17:38,948 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:38,949 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/77fb254c479a4ec980b3068634515720 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77fb254c479a4ec980b3068634515720 2024-11-14T07:17:38,955 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/A of 17b20244019df3f0f04e41107ed48efc into 77fb254c479a4ec980b3068634515720(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:38,955 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:38,955 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/A, priority=13, startTime=1731568658492; duration=0sec 2024-11-14T07:17:38,955 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:38,955 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568718962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568718962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568718972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:38,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:39,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ceb6dd3f86ee4fe59b406e247832c349_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ceb6dd3f86ee4fe59b406e247832c349_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:39,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/ec6d3d01ba6f4dd791e6a794f6677b1f, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:39,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/ec6d3d01ba6f4dd791e6a794f6677b1f is 175, key is test_row_0/A:col10/1731568657656/Put/seqid=0 2024-11-14T07:17:39,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742210_1386 (size=31255) 2024-11-14T07:17:39,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/ec6d3d01ba6f4dd791e6a794f6677b1f 2024-11-14T07:17:39,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/153a3c8bf3254981a12ae8b82ee42806 is 50, key is test_row_0/B:col10/1731568657656/Put/seqid=0 2024-11-14T07:17:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742211_1387 (size=12301) 2024-11-14T07:17:39,045 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/153a3c8bf3254981a12ae8b82ee42806 2024-11-14T07:17:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/732a9fe42e89412b901b513a7031afd0 is 50, key is test_row_0/C:col10/1731568657656/Put/seqid=0 2024-11-14T07:17:39,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742212_1388 (size=12301) 2024-11-14T07:17:39,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/732a9fe42e89412b901b513a7031afd0 2024-11-14T07:17:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:17:39,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/ec6d3d01ba6f4dd791e6a794f6677b1f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ec6d3d01ba6f4dd791e6a794f6677b1f 2024-11-14T07:17:39,092 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ec6d3d01ba6f4dd791e6a794f6677b1f, entries=150, sequenceid=269, filesize=30.5 K 2024-11-14T07:17:39,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/153a3c8bf3254981a12ae8b82ee42806 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/153a3c8bf3254981a12ae8b82ee42806 2024-11-14T07:17:39,097 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/153a3c8bf3254981a12ae8b82ee42806, entries=150, sequenceid=269, filesize=12.0 K 2024-11-14T07:17:39,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/732a9fe42e89412b901b513a7031afd0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/732a9fe42e89412b901b513a7031afd0 2024-11-14T07:17:39,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/732a9fe42e89412b901b513a7031afd0, entries=150, sequenceid=269, filesize=12.0 K 2024-11-14T07:17:39,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 17b20244019df3f0f04e41107ed48efc in 543ms, sequenceid=269, compaction requested=true 2024-11-14T07:17:39,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:39,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T07:17:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T07:17:39,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T07:17:39,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1570 sec 2024-11-14T07:17:39,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.1640 sec 2024-11-14T07:17:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:39,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T07:17:39,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:39,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:39,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:39,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:39,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:39,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:17:39,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e5ca082febd443ef927edc39525be4be_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568658846/Put/seqid=0 2024-11-14T07:17:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742213_1389 (size=14994) 2024-11-14T07:17:39,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:39,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e5ca082febd443ef927edc39525be4be_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e5ca082febd443ef927edc39525be4be_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:39,205 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:39,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1 is 175, key is test_row_0/A:col10/1731568658846/Put/seqid=0 2024-11-14T07:17:39,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742214_1390 (size=39949) 2024-11-14T07:17:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568719200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568719208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568719209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568719322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568719322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568719332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568719542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568719542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568719552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1 2024-11-14T07:17:39,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/acd23a269e494bb1a61808ba889de1c2 is 50, key is test_row_0/B:col10/1731568658846/Put/seqid=0 2024-11-14T07:17:39,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742215_1391 (size=12301) 2024-11-14T07:17:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568719852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568719852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:39,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568719872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:40,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/acd23a269e494bb1a61808ba889de1c2 2024-11-14T07:17:40,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/e14c8066172144b6a98c0e3e5187f9a8 is 50, key is test_row_0/C:col10/1731568658846/Put/seqid=0 2024-11-14T07:17:40,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742216_1392 (size=12301) 2024-11-14T07:17:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:17:40,092 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:40,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T07:17:40,094 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:17:40,095 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:40,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:17:40,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T07:17:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:40,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568720362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:40,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568720374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:40,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T07:17:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:40,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568720392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:17:40,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/e14c8066172144b6a98c0e3e5187f9a8 2024-11-14T07:17:40,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1 2024-11-14T07:17:40,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1, entries=200, sequenceid=293, filesize=39.0 K 2024-11-14T07:17:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/acd23a269e494bb1a61808ba889de1c2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/acd23a269e494bb1a61808ba889de1c2 2024-11-14T07:17:40,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/acd23a269e494bb1a61808ba889de1c2, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T07:17:40,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/e14c8066172144b6a98c0e3e5187f9a8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e14c8066172144b6a98c0e3e5187f9a8 2024-11-14T07:17:40,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e14c8066172144b6a98c0e3e5187f9a8, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T07:17:40,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 17b20244019df3f0f04e41107ed48efc in 1265ms, sequenceid=293, compaction requested=true 2024-11-14T07:17:40,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:40,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:40,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:40,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:40,453 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:40,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:40,453 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:40,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:40,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:40,454 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:40,455 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/B is initiating minor compaction (all files) 2024-11-14T07:17:40,455 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/B in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,455 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d3a79775127e45debe62952d1442476b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/153a3c8bf3254981a12ae8b82ee42806, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/acd23a269e494bb1a61808ba889de1c2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=36.4 K 2024-11-14T07:17:40,455 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:40,455 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/A is initiating minor compaction (all files) 2024-11-14T07:17:40,455 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/A in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,455 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77fb254c479a4ec980b3068634515720, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ec6d3d01ba6f4dd791e6a794f6677b1f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=100.4 K 2024-11-14T07:17:40,455 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,455 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77fb254c479a4ec980b3068634515720, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ec6d3d01ba6f4dd791e6a794f6677b1f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1] 2024-11-14T07:17:40,456 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a79775127e45debe62952d1442476b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731568657581 2024-11-14T07:17:40,456 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77fb254c479a4ec980b3068634515720, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731568657581 2024-11-14T07:17:40,456 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 153a3c8bf3254981a12ae8b82ee42806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731568657635 2024-11-14T07:17:40,456 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec6d3d01ba6f4dd791e6a794f6677b1f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731568657635 2024-11-14T07:17:40,456 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting acd23a269e494bb1a61808ba889de1c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568658846 2024-11-14T07:17:40,457 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd0c01e4fdb4a90bbb8f03e97f0aca1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568658846 2024-11-14T07:17:40,478 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:40,479 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/7dd0f4aa4b5c49a4a53de8fd1218eff5 is 50, key is test_row_0/B:col10/1731568658846/Put/seqid=0 2024-11-14T07:17:40,480 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:40,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742217_1393 (size=12915) 2024-11-14T07:17:40,509 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d5687ccedbc549b8a0babae9447ec911_17b20244019df3f0f04e41107ed48efc store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:40,511 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d5687ccedbc549b8a0babae9447ec911_17b20244019df3f0f04e41107ed48efc, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:40,512 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d5687ccedbc549b8a0babae9447ec911_17b20244019df3f0f04e41107ed48efc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:40,516 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/7dd0f4aa4b5c49a4a53de8fd1218eff5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/7dd0f4aa4b5c49a4a53de8fd1218eff5 2024-11-14T07:17:40,519 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/B of 17b20244019df3f0f04e41107ed48efc into 7dd0f4aa4b5c49a4a53de8fd1218eff5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:40,519 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:40,519 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/B, priority=13, startTime=1731568660453; duration=0sec 2024-11-14T07:17:40,519 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:40,519 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:40,520 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:17:40,520 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:17:40,520 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/C is initiating minor compaction (all files) 2024-11-14T07:17:40,520 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/C in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,520 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/14f54455ab6e4a8185262ded7804f7a2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/0f5b428681384ab4ba160305361ec455, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/732a9fe42e89412b901b513a7031afd0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e14c8066172144b6a98c0e3e5187f9a8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=48.2 K 2024-11-14T07:17:40,521 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 14f54455ab6e4a8185262ded7804f7a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731568655286 2024-11-14T07:17:40,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742218_1394 (size=4469) 2024-11-14T07:17:40,521 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f5b428681384ab4ba160305361ec455, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731568657581 2024-11-14T07:17:40,521 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 732a9fe42e89412b901b513a7031afd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731568657635 2024-11-14T07:17:40,522 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e14c8066172144b6a98c0e3e5187f9a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568658846 2024-11-14T07:17:40,529 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:40,529 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/345240fb30c948f2a43009d90cf348eb is 50, key is test_row_0/C:col10/1731568658846/Put/seqid=0 2024-11-14T07:17:40,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742219_1395 (size=12881) 2024-11-14T07:17:40,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T07:17:40,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:40,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:17:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:40,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fc3e13b4459342338afa19d399e0328b_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568659207/Put/seqid=0 2024-11-14T07:17:40,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742220_1396 (size=12454) 2024-11-14T07:17:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:17:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:40,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:40,922 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:40,923 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/ad523501cea94f76994bbf7a99b69108 is 175, key is test_row_0/A:col10/1731568658846/Put/seqid=0 2024-11-14T07:17:40,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742221_1397 (size=31869) 2024-11-14T07:17:40,938 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/345240fb30c948f2a43009d90cf348eb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/345240fb30c948f2a43009d90cf348eb 2024-11-14T07:17:40,942 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/C of 17b20244019df3f0f04e41107ed48efc into 345240fb30c948f2a43009d90cf348eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:40,942 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:40,942 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/C, priority=12, startTime=1731568660453; duration=0sec 2024-11-14T07:17:40,943 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:40,943 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:40,970 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fc3e13b4459342338afa19d399e0328b_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fc3e13b4459342338afa19d399e0328b_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:40,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/270da8729c1d4382b278ae0d48a88557, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:40,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/270da8729c1d4382b278ae0d48a88557 is 175, key is test_row_0/A:col10/1731568659207/Put/seqid=0 2024-11-14T07:17:40,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742222_1398 (size=31255) 2024-11-14T07:17:41,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568660989, max timestamp=1731568661008Num uniques 859; 2024-11-14T07:17:41,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568660989, max timestamp=1731568661008Num uniques 859; 2024-11-14T07:17:41,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568660989, max timestamp=1731568661008Num uniques 859; 2024-11-14T07:17:41,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:17:41,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:17:41,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:17:41,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568721134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:17:41,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568721251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,331 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/ad523501cea94f76994bbf7a99b69108 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ad523501cea94f76994bbf7a99b69108 2024-11-14T07:17:41,335 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/A of 17b20244019df3f0f04e41107ed48efc into ad523501cea94f76994bbf7a99b69108(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:41,335 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:41,335 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/A, priority=13, startTime=1731568660453; duration=0sec 2024-11-14T07:17:41,335 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:41,335 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:41,377 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/270da8729c1d4382b278ae0d48a88557 2024-11-14T07:17:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ac5e9813476d46299d8c96e4e3ece518 is 50, key is test_row_0/B:col10/1731568659207/Put/seqid=0 2024-11-14T07:17:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568721382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742223_1399 (size=12301) 2024-11-14T07:17:41,393 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,394 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ac5e9813476d46299d8c96e4e3ece518 2024-11-14T07:17:41,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568721392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,397 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/cbbada88161a46558965c440bff4cf9b is 50, key is test_row_0/C:col10/1731568659207/Put/seqid=0 2024-11-14T07:17:41,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742224_1400 (size=12301) 2024-11-14T07:17:41,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/cbbada88161a46558965c440bff4cf9b 2024-11-14T07:17:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/270da8729c1d4382b278ae0d48a88557 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/270da8729c1d4382b278ae0d48a88557 2024-11-14T07:17:41,418 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/270da8729c1d4382b278ae0d48a88557, entries=150, sequenceid=304, filesize=30.5 K 2024-11-14T07:17:41,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/ac5e9813476d46299d8c96e4e3ece518 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ac5e9813476d46299d8c96e4e3ece518 2024-11-14T07:17:41,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ac5e9813476d46299d8c96e4e3ece518, entries=150, sequenceid=304, filesize=12.0 K 2024-11-14T07:17:41,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/cbbada88161a46558965c440bff4cf9b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/cbbada88161a46558965c440bff4cf9b 2024-11-14T07:17:41,439 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/cbbada88161a46558965c440bff4cf9b, entries=150, sequenceid=304, filesize=12.0 K 2024-11-14T07:17:41,440 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 17b20244019df3f0f04e41107ed48efc in 888ms, sequenceid=304, compaction requested=false 2024-11-14T07:17:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T07:17:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T07:17:41,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-14T07:17:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:41,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T07:17:41,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3480 sec 2024-11-14T07:17:41,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.3540 sec 2024-11-14T07:17:41,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143216535fc5ba495e822e3b8ff38b388c_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568661133/Put/seqid=0 2024-11-14T07:17:41,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742225_1401 (size=14994) 2024-11-14T07:17:41,483 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:41,491 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143216535fc5ba495e822e3b8ff38b388c_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143216535fc5ba495e822e3b8ff38b388c_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:41,492 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9051f5d2d51a4623bfd449407695baed, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:41,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9051f5d2d51a4623bfd449407695baed is 175, key is test_row_0/A:col10/1731568661133/Put/seqid=0 2024-11-14T07:17:41,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568721491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742226_1402 (size=39949) 2024-11-14T07:17:41,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568721503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568721622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568721631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568721842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568721852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:41,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9051f5d2d51a4623bfd449407695baed 2024-11-14T07:17:41,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/49b7cbe7be234c5b8971b17305dc7e3c is 50, key is test_row_0/B:col10/1731568661133/Put/seqid=0 2024-11-14T07:17:41,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742227_1403 (size=12301) 2024-11-14T07:17:42,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568722162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:42,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568722162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:17:42,252 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:42,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-14T07:17:42,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:42,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:42,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:17:42,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/49b7cbe7be234c5b8971b17305dc7e3c 2024-11-14T07:17:42,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/46d8afdd4887430a9a39ba5dc814fe8f is 50, key is test_row_0/C:col10/1731568661133/Put/seqid=0 2024-11-14T07:17:42,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742228_1404 (size=12301) 2024-11-14T07:17:42,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/46d8afdd4887430a9a39ba5dc814fe8f 2024-11-14T07:17:42,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/9051f5d2d51a4623bfd449407695baed as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9051f5d2d51a4623bfd449407695baed 2024-11-14T07:17:42,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9051f5d2d51a4623bfd449407695baed, entries=200, sequenceid=344, filesize=39.0 K 2024-11-14T07:17:42,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/49b7cbe7be234c5b8971b17305dc7e3c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/49b7cbe7be234c5b8971b17305dc7e3c 2024-11-14T07:17:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/49b7cbe7be234c5b8971b17305dc7e3c, entries=150, sequenceid=344, filesize=12.0 K 2024-11-14T07:17:42,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/46d8afdd4887430a9a39ba5dc814fe8f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46d8afdd4887430a9a39ba5dc814fe8f 2024-11-14T07:17:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:17:42,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46d8afdd4887430a9a39ba5dc814fe8f, entries=150, sequenceid=344, filesize=12.0 K 2024-11-14T07:17:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 17b20244019df3f0f04e41107ed48efc in 934ms, sequenceid=344, compaction requested=true 2024-11-14T07:17:42,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:42,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:42,376 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:42,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:42,376 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:42,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:42,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:42,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:42,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,377 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:42,377 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/A is initiating minor compaction (all files) 2024-11-14T07:17:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,377 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/A in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:42,377 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ad523501cea94f76994bbf7a99b69108, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/270da8729c1d4382b278ae0d48a88557, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9051f5d2d51a4623bfd449407695baed] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=100.7 K 2024-11-14T07:17:42,377 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:42,378 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ad523501cea94f76994bbf7a99b69108, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/270da8729c1d4382b278ae0d48a88557, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9051f5d2d51a4623bfd449407695baed] 2024-11-14T07:17:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,378 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,378 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/B is initiating minor compaction (all files) 2024-11-14T07:17:42,378 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/B in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,378 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/7dd0f4aa4b5c49a4a53de8fd1218eff5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ac5e9813476d46299d8c96e4e3ece518, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/49b7cbe7be234c5b8971b17305dc7e3c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=36.6 K 2024-11-14T07:17:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,378 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad523501cea94f76994bbf7a99b69108, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568658846 2024-11-14T07:17:42,378 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd0f4aa4b5c49a4a53de8fd1218eff5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568658846 2024-11-14T07:17:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,379 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5e9813476d46299d8c96e4e3ece518, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731568659190 2024-11-14T07:17:42,379 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 270da8729c1d4382b278ae0d48a88557, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731568659190 2024-11-14T07:17:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,379 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9051f5d2d51a4623bfd449407695baed, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731568661090 2024-11-14T07:17:42,379 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b7cbe7be234c5b8971b17305dc7e3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731568661090 2024-11-14T07:17:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,384 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,385 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,386 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b2038ccf3cfe44c38ad9f184c961b89a_17b20244019df3f0f04e41107ed48efc store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,386 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/f0f2e1691e744946a01309b9102626c1 is 50, key is test_row_0/B:col10/1731568661133/Put/seqid=0 2024-11-14T07:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,388 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b2038ccf3cfe44c38ad9f184c961b89a_17b20244019df3f0f04e41107ed48efc, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:42,388 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b2038ccf3cfe44c38ad9f184c961b89a_17b20244019df3f0f04e41107ed48efc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:17:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:42,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T07:17:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742229_1405 (size=13017) 2024-11-14T07:17:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742230_1406 (size=4469) 2024-11-14T07:17:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147fe46c3daa494d109730b1a869ffdd6e_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_1/A:col10/1731568661501/Put/seqid=0 2024-11-14T07:17:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742231_1407 (size=9914) 2024-11-14T07:17:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147fe46c3daa494d109730b1a869ffdd6e_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147fe46c3daa494d109730b1a869ffdd6e_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/869d2331112d417091c7a5f3243f7b5c, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/869d2331112d417091c7a5f3243f7b5c is 175, key is test_row_1/A:col10/1731568661501/Put/seqid=0 2024-11-14T07:17:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742232_1408 (size=22561) 2024-11-14T07:17:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:17:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:42,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:42,835 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/f0f2e1691e744946a01309b9102626c1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f0f2e1691e744946a01309b9102626c1 2024-11-14T07:17:42,839 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#A#compaction#336 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:42,839 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/18604d3695134840a5b8bf66ecd3a390 is 175, key is test_row_0/A:col10/1731568661133/Put/seqid=0 2024-11-14T07:17:42,840 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/B of 17b20244019df3f0f04e41107ed48efc into f0f2e1691e744946a01309b9102626c1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:42,840 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:42,840 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/B, priority=13, startTime=1731568662376; duration=0sec 2024-11-14T07:17:42,841 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:42,841 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:42,841 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:42,843 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:42,843 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/C is initiating minor compaction (all files) 2024-11-14T07:17:42,843 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/C in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:42,843 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/345240fb30c948f2a43009d90cf348eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/cbbada88161a46558965c440bff4cf9b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46d8afdd4887430a9a39ba5dc814fe8f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=36.6 K 2024-11-14T07:17:42,846 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 345240fb30c948f2a43009d90cf348eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568658846 2024-11-14T07:17:42,846 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbada88161a46558965c440bff4cf9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731568659190 2024-11-14T07:17:42,847 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d8afdd4887430a9a39ba5dc814fe8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731568661090 2024-11-14T07:17:42,853 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:42,853 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/5d248fa1a5b44ec79c356cee7a375db8 is 50, key is test_row_0/C:col10/1731568661133/Put/seqid=0 2024-11-14T07:17:42,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742233_1409 (size=31971) 2024-11-14T07:17:42,866 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/18604d3695134840a5b8bf66ecd3a390 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/18604d3695134840a5b8bf66ecd3a390 2024-11-14T07:17:42,870 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/A of 17b20244019df3f0f04e41107ed48efc into 18604d3695134840a5b8bf66ecd3a390(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:42,870 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:42,870 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/A, priority=13, startTime=1731568662375; duration=0sec 2024-11-14T07:17:42,870 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:42,870 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:42,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742234_1410 (size=12983) 2024-11-14T07:17:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:17:42,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568662857, max timestamp=1731568662889Num uniques 859; 2024-11-14T07:17:42,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568662857, max timestamp=1731568662889Num uniques 859; 2024-11-14T07:17:42,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:42,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:42,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568662857, max timestamp=1731568662889Num uniques 859; 2024-11-14T07:17:42,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:17:42,899 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/869d2331112d417091c7a5f3243f7b5c 2024-11-14T07:17:42,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/87ae98710b884ede8bb8fc5b6e1ad2ef is 50, key is test_row_1/B:col10/1731568661501/Put/seqid=0 2024-11-14T07:17:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742235_1411 (size=9857) 2024-11-14T07:17:43,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568723002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568723003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568723111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568723122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,278 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/5d248fa1a5b44ec79c356cee7a375db8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/5d248fa1a5b44ec79c356cee7a375db8 2024-11-14T07:17:43,282 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/C of 17b20244019df3f0f04e41107ed48efc into 5d248fa1a5b44ec79c356cee7a375db8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:43,282 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:43,282 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/C, priority=13, startTime=1731568662376; duration=0sec 2024-11-14T07:17:43,283 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:43,283 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:43,334 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/87ae98710b884ede8bb8fc5b6e1ad2ef 2024-11-14T07:17:43,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568723331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568723331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/42227423a18245898e87df8099531a96 is 50, key is test_row_1/C:col10/1731568661501/Put/seqid=0 2024-11-14T07:17:43,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742236_1412 (size=9857) 2024-11-14T07:17:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:17:43,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41356 deadline: 1731568723402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,407 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41348 deadline: 1731568723411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,421 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568723642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568723642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41342 deadline: 1731568723722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:43,724 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 17b20244019df3f0f04e41107ed48efc of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:17:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:43,743 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/42227423a18245898e87df8099531a96 2024-11-14T07:17:43,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/869d2331112d417091c7a5f3243f7b5c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/869d2331112d417091c7a5f3243f7b5c 2024-11-14T07:17:43,760 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/869d2331112d417091c7a5f3243f7b5c, entries=100, sequenceid=353, filesize=22.0 K 2024-11-14T07:17:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/87ae98710b884ede8bb8fc5b6e1ad2ef as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/87ae98710b884ede8bb8fc5b6e1ad2ef 2024-11-14T07:17:43,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/87ae98710b884ede8bb8fc5b6e1ad2ef, entries=100, sequenceid=353, filesize=9.6 K 2024-11-14T07:17:43,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/42227423a18245898e87df8099531a96 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/42227423a18245898e87df8099531a96 2024-11-14T07:17:43,767 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/42227423a18245898e87df8099531a96, entries=100, sequenceid=353, filesize=9.6 K 2024-11-14T07:17:43,768 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 17b20244019df3f0f04e41107ed48efc in 1358ms, sequenceid=353, compaction requested=false 2024-11-14T07:17:43,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:43,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:43,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-14T07:17:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-14T07:17:43,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T07:17:43,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5120 sec 2024-11-14T07:17:43,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5190 sec 2024-11-14T07:17:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:44,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-11-14T07:17:44,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:44,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:44,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:44,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:44,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:44,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:17:44,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144cf1248b4df24e4097b8d5e305f53644_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568664160/Put/seqid=0 2024-11-14T07:17:44,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742237_1413 (size=17534) 2024-11-14T07:17:44,173 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:44,177 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144cf1248b4df24e4097b8d5e305f53644_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144cf1248b4df24e4097b8d5e305f53644_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:44,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/679ac3ed7cfb49dc94c135b35b9c0e37, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:44,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/679ac3ed7cfb49dc94c135b35b9c0e37 is 175, key is test_row_0/A:col10/1731568664160/Put/seqid=0 2024-11-14T07:17:44,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568724175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742238_1414 (size=48639) 2024-11-14T07:17:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/679ac3ed7cfb49dc94c135b35b9c0e37 2024-11-14T07:17:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568724185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/6d5e485be59a4c6b80412f47a6910787 is 50, key is test_row_0/B:col10/1731568664160/Put/seqid=0 2024-11-14T07:17:44,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742239_1415 (size=12301) 2024-11-14T07:17:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568724292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568724302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:17:44,412 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:44,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-14T07:17:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:17:44,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:44,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:44,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:44,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568724512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:17:44,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568724522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:17:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:44,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/6d5e485be59a4c6b80412f47a6910787 2024-11-14T07:17:44,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f23f7153b7104039a3b96ee81f08bd74 is 50, key is test_row_0/C:col10/1731568664160/Put/seqid=0 2024-11-14T07:17:44,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742240_1416 (size=12301) 2024-11-14T07:17:44,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:17:44,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:44,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:44,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:44,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:17:44,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1731568724821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41370 deadline: 1731568724832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:17:44,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:17:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:44,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:44,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:45,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f23f7153b7104039a3b96ee81f08bd74 2024-11-14T07:17:45,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/679ac3ed7cfb49dc94c135b35b9c0e37 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/679ac3ed7cfb49dc94c135b35b9c0e37 2024-11-14T07:17:45,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/679ac3ed7cfb49dc94c135b35b9c0e37, entries=250, sequenceid=395, filesize=47.5 K 2024-11-14T07:17:45,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/6d5e485be59a4c6b80412f47a6910787 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/6d5e485be59a4c6b80412f47a6910787 2024-11-14T07:17:45,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:17:45,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:45,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. as already flushing 2024-11-14T07:17:45,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:45,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:45,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:45,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/6d5e485be59a4c6b80412f47a6910787, entries=150, sequenceid=395, filesize=12.0 K 2024-11-14T07:17:45,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f23f7153b7104039a3b96ee81f08bd74 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f23f7153b7104039a3b96ee81f08bd74 2024-11-14T07:17:45,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f23f7153b7104039a3b96ee81f08bd74, entries=150, sequenceid=395, filesize=12.0 K 2024-11-14T07:17:45,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for 17b20244019df3f0f04e41107ed48efc in 870ms, sequenceid=395, compaction requested=true 2024-11-14T07:17:45,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:45,031 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:17:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:45,032 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:17:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b20244019df3f0f04e41107ed48efc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:17:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:45,033 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:45,033 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/B is initiating minor compaction (all files) 2024-11-14T07:17:45,033 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/B in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:45,033 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f0f2e1691e744946a01309b9102626c1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/87ae98710b884ede8bb8fc5b6e1ad2ef, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/6d5e485be59a4c6b80412f47a6910787] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=34.4 K 2024-11-14T07:17:45,033 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:45,033 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/A is initiating minor compaction (all files) 2024-11-14T07:17:45,033 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/A in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:45,033 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/18604d3695134840a5b8bf66ecd3a390, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/869d2331112d417091c7a5f3243f7b5c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/679ac3ed7cfb49dc94c135b35b9c0e37] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=100.8 K 2024-11-14T07:17:45,033 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:45,033 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/18604d3695134840a5b8bf66ecd3a390, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/869d2331112d417091c7a5f3243f7b5c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/679ac3ed7cfb49dc94c135b35b9c0e37] 2024-11-14T07:17:45,033 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f0f2e1691e744946a01309b9102626c1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731568661090 2024-11-14T07:17:45,034 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18604d3695134840a5b8bf66ecd3a390, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731568661090 2024-11-14T07:17:45,034 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ae98710b884ede8bb8fc5b6e1ad2ef, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731568661470 2024-11-14T07:17:45,034 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 869d2331112d417091c7a5f3243f7b5c, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731568661470 2024-11-14T07:17:45,035 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 679ac3ed7cfb49dc94c135b35b9c0e37, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731568662970 2024-11-14T07:17:45,035 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5e485be59a4c6b80412f47a6910787, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731568662970 2024-11-14T07:17:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:17:45,043 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#B#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:45,043 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/0ea69f693b934c4a95187870ad958d35 is 50, key is test_row_0/B:col10/1731568664160/Put/seqid=0 2024-11-14T07:17:45,046 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:45,049 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d5ef8dffc57844f8be5099d74acb34d2_17b20244019df3f0f04e41107ed48efc store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:45,050 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d5ef8dffc57844f8be5099d74acb34d2_17b20244019df3f0f04e41107ed48efc, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:45,050 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d5ef8dffc57844f8be5099d74acb34d2_17b20244019df3f0f04e41107ed48efc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:45,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742241_1417 (size=13119) 2024-11-14T07:17:45,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742242_1418 (size=4469) 2024-11-14T07:17:45,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:17:45,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:45,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T07:17:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114713ef52140eb433d9a468690ae298136_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568664174/Put/seqid=0 2024-11-14T07:17:45,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742243_1419 (size=12454) 2024-11-14T07:17:45,325 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-14T07:17:45,325 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:45,325 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,325 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:45,327 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-14T07:17:45,327 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:45,327 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,327 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:45,328 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-14T07:17:45,328 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:45,328 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,328 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:45,331 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-14T07:17:45,331 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:45,331 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,331 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:45,332 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-11-14T07:17:45,332 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:45,333 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,333 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:45,343 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-11-14T07:17:45,343 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:45,343 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,343 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:45,353 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-11-14T07:17:45,353 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:45,353 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,353 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:45,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:45,458 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/0ea69f693b934c4a95187870ad958d35 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ea69f693b934c4a95187870ad958d35 2024-11-14T07:17:45,461 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#A#compaction#345 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:45,461 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/4be66c00f5c54678b04c30dd53deb064 is 175, key is test_row_0/A:col10/1731568664160/Put/seqid=0 2024-11-14T07:17:45,461 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/B of 17b20244019df3f0f04e41107ed48efc into 0ea69f693b934c4a95187870ad958d35(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:45,461 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:45,461 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/B, priority=13, startTime=1731568665032; duration=0sec 2024-11-14T07:17:45,461 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:17:45,461 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:B 2024-11-14T07:17:45,462 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:17:45,462 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:17:45,462 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 17b20244019df3f0f04e41107ed48efc/C is initiating minor compaction (all files) 2024-11-14T07:17:45,462 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b20244019df3f0f04e41107ed48efc/C in TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:45,462 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/5d248fa1a5b44ec79c356cee7a375db8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/42227423a18245898e87df8099531a96, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f23f7153b7104039a3b96ee81f08bd74] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp, totalSize=34.3 K 2024-11-14T07:17:45,463 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d248fa1a5b44ec79c356cee7a375db8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731568661090 2024-11-14T07:17:45,463 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 42227423a18245898e87df8099531a96, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731568661470 2024-11-14T07:17:45,463 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f23f7153b7104039a3b96ee81f08bd74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731568662970 2024-11-14T07:17:45,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742244_1420 (size=32073) 2024-11-14T07:17:45,468 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b20244019df3f0f04e41107ed48efc#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:17:45,468 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/b4b077882c974633b26d5317938303a5 is 50, key is test_row_0/C:col10/1731568664160/Put/seqid=0 2024-11-14T07:17:45,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742245_1421 (size=13085) 2024-11-14T07:17:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:17:45,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:45,597 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114713ef52140eb433d9a468690ae298136_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114713ef52140eb433d9a468690ae298136_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a12ef7a61b80422389c396d9f792e8d1, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a12ef7a61b80422389c396d9f792e8d1 is 175, key is test_row_0/A:col10/1731568664174/Put/seqid=0 2024-11-14T07:17:45,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742246_1422 (size=31255) 2024-11-14T07:17:45,868 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/4be66c00f5c54678b04c30dd53deb064 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/4be66c00f5c54678b04c30dd53deb064 2024-11-14T07:17:45,871 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/A of 17b20244019df3f0f04e41107ed48efc into 4be66c00f5c54678b04c30dd53deb064(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:45,872 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:45,872 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/A, priority=13, startTime=1731568665031; duration=0sec 2024-11-14T07:17:45,872 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:45,872 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:A 2024-11-14T07:17:45,875 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/b4b077882c974633b26d5317938303a5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b4b077882c974633b26d5317938303a5 2024-11-14T07:17:45,879 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b20244019df3f0f04e41107ed48efc/C of 17b20244019df3f0f04e41107ed48efc into b4b077882c974633b26d5317938303a5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:17:45,879 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:45,879 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., storeName=17b20244019df3f0f04e41107ed48efc/C, priority=13, startTime=1731568665032; duration=0sec 2024-11-14T07:17:45,879 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:17:45,879 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b20244019df3f0f04e41107ed48efc:C 2024-11-14T07:17:46,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a12ef7a61b80422389c396d9f792e8d1 2024-11-14T07:17:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/bbea26c43bb14115a35260a6f363818a is 50, key is test_row_0/B:col10/1731568664174/Put/seqid=0 2024-11-14T07:17:46,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742247_1423 (size=12301) 2024-11-14T07:17:46,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/bbea26c43bb14115a35260a6f363818a 2024-11-14T07:17:46,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f895b7933dcb4749a6fb7c08de6e735f is 50, key is test_row_0/C:col10/1731568664174/Put/seqid=0 2024-11-14T07:17:46,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742248_1424 (size=12301) 2024-11-14T07:17:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:17:46,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f895b7933dcb4749a6fb7c08de6e735f 2024-11-14T07:17:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/a12ef7a61b80422389c396d9f792e8d1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a12ef7a61b80422389c396d9f792e8d1 2024-11-14T07:17:46,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a12ef7a61b80422389c396d9f792e8d1, entries=150, sequenceid=402, filesize=30.5 K 2024-11-14T07:17:46,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/bbea26c43bb14115a35260a6f363818a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/bbea26c43bb14115a35260a6f363818a 2024-11-14T07:17:46,827 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/bbea26c43bb14115a35260a6f363818a, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T07:17:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/f895b7933dcb4749a6fb7c08de6e735f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f895b7933dcb4749a6fb7c08de6e735f 2024-11-14T07:17:46,830 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f895b7933dcb4749a6fb7c08de6e735f, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T07:17:46,831 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=13.42 KB/13740 for 17b20244019df3f0f04e41107ed48efc in 1653ms, sequenceid=402, compaction requested=false 2024-11-14T07:17:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 17b20244019df3f0f04e41107ed48efc: 2024-11-14T07:17:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-14T07:17:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-14T07:17:46,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T07:17:46,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4170 sec 2024-11-14T07:17:46,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.4210 sec 2024-11-14T07:17:47,413 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-14T07:17:47,413 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:47,413 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:47,413 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:47,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:47,454 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-11-14T07:17:47,454 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:47,454 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:47,454 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:47,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:48,497 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:17:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:17:48,583 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:52,616 WARN [regionserver/c7aa6e2790b3:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-14T07:17:52,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9e894732083241f7b93c42a884bc2c5c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/027ff4355c4346b28d4435f74a7ab431, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77db160c597b486c8f0783ed7505663f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/479be62f89a244f38851d48d94765326, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/01ae4df667aa48d2b44fdf1f41febee7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a554e1c6783c4f1eba535fd984ce82d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/bbef56b9344f40caa40f41400eaf2673, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/7e0cf78d8e864cb592153bb8eea3700a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a3cbbe627cdb440ba142a84efa08a009, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/24a50aae1f9440d5b3e90fbd6c7bd689, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/6de37254321348d8bb6f62606cf0be70, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/febb1fa9106948fa8e062c9d4747d21a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/2888ded141d240ee97210d041b9ff420, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/adb59187890b48e4becb38a14d5f1e49, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/de4b9f530688434a9ec534906a6fd17c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77fb254c479a4ec980b3068634515720, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ec6d3d01ba6f4dd791e6a794f6677b1f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ad523501cea94f76994bbf7a99b69108, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/270da8729c1d4382b278ae0d48a88557, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9051f5d2d51a4623bfd449407695baed, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/18604d3695134840a5b8bf66ecd3a390, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/869d2331112d417091c7a5f3243f7b5c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/679ac3ed7cfb49dc94c135b35b9c0e37] to archive 2024-11-14T07:17:52,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:17:52,620 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9e894732083241f7b93c42a884bc2c5c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9e894732083241f7b93c42a884bc2c5c 2024-11-14T07:17:52,621 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/027ff4355c4346b28d4435f74a7ab431 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/027ff4355c4346b28d4435f74a7ab431 2024-11-14T07:17:52,622 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77db160c597b486c8f0783ed7505663f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77db160c597b486c8f0783ed7505663f 2024-11-14T07:17:52,623 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/479be62f89a244f38851d48d94765326 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/479be62f89a244f38851d48d94765326 2024-11-14T07:17:52,624 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/01ae4df667aa48d2b44fdf1f41febee7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/01ae4df667aa48d2b44fdf1f41febee7 2024-11-14T07:17:52,624 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a554e1c6783c4f1eba535fd984ce82d8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a554e1c6783c4f1eba535fd984ce82d8 2024-11-14T07:17:52,625 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/bbef56b9344f40caa40f41400eaf2673 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/bbef56b9344f40caa40f41400eaf2673 2024-11-14T07:17:52,626 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/7e0cf78d8e864cb592153bb8eea3700a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/7e0cf78d8e864cb592153bb8eea3700a 2024-11-14T07:17:52,627 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a3cbbe627cdb440ba142a84efa08a009 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a3cbbe627cdb440ba142a84efa08a009 2024-11-14T07:17:52,628 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/24a50aae1f9440d5b3e90fbd6c7bd689 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/24a50aae1f9440d5b3e90fbd6c7bd689 2024-11-14T07:17:52,629 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/6de37254321348d8bb6f62606cf0be70 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/6de37254321348d8bb6f62606cf0be70 2024-11-14T07:17:52,629 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/febb1fa9106948fa8e062c9d4747d21a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/febb1fa9106948fa8e062c9d4747d21a 2024-11-14T07:17:52,631 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/2888ded141d240ee97210d041b9ff420 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/2888ded141d240ee97210d041b9ff420 2024-11-14T07:17:52,632 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/adb59187890b48e4becb38a14d5f1e49 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/adb59187890b48e4becb38a14d5f1e49 2024-11-14T07:17:52,633 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/de4b9f530688434a9ec534906a6fd17c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/de4b9f530688434a9ec534906a6fd17c 2024-11-14T07:17:52,634 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77fb254c479a4ec980b3068634515720 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/77fb254c479a4ec980b3068634515720 2024-11-14T07:17:52,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ec6d3d01ba6f4dd791e6a794f6677b1f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ec6d3d01ba6f4dd791e6a794f6677b1f 2024-11-14T07:17:52,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/3bd0c01e4fdb4a90bbb8f03e97f0aca1 2024-11-14T07:17:52,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ad523501cea94f76994bbf7a99b69108 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/ad523501cea94f76994bbf7a99b69108 2024-11-14T07:17:52,637 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/270da8729c1d4382b278ae0d48a88557 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/270da8729c1d4382b278ae0d48a88557 2024-11-14T07:17:52,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9051f5d2d51a4623bfd449407695baed to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/9051f5d2d51a4623bfd449407695baed 2024-11-14T07:17:52,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/18604d3695134840a5b8bf66ecd3a390 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/18604d3695134840a5b8bf66ecd3a390 2024-11-14T07:17:52,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/869d2331112d417091c7a5f3243f7b5c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/869d2331112d417091c7a5f3243f7b5c 2024-11-14T07:17:52,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/679ac3ed7cfb49dc94c135b35b9c0e37 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/679ac3ed7cfb49dc94c135b35b9c0e37 2024-11-14T07:17:52,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9b9aea32f344499ca13956bafdb3d64e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/cd612fcec7a44fbfac5216d8f97ae207, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/74a5d8ede6e84aeebd9e8f31ad800054, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/fc75405d2acb48358f127454b43a0e72, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9eaf8a76d524462681e615b3fc30fe04, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ca5d32e344e04171a98f3b8330a5b771, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f56d1a63f8e14d758859ec309ffeef20, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ae35b3bcb17403b97d903b77bfbddb3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d2b914c40d9647149327c8dc252b439a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/2eaea9ae3fc34e478941985ee7cb5c83, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ab252864b2c04276af660ec77c7e1a2f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/a0f8266a85894d5a96de5ef221deb968, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/374c152a1d174f8fa2374d915d15e0cc, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/8d21cfa824844babbba9392540f46d38, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d3a79775127e45debe62952d1442476b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d16e24e36bab423dbea55cd16deabd83, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/153a3c8bf3254981a12ae8b82ee42806, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/7dd0f4aa4b5c49a4a53de8fd1218eff5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/acd23a269e494bb1a61808ba889de1c2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ac5e9813476d46299d8c96e4e3ece518, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f0f2e1691e744946a01309b9102626c1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/49b7cbe7be234c5b8971b17305dc7e3c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/87ae98710b884ede8bb8fc5b6e1ad2ef, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/6d5e485be59a4c6b80412f47a6910787] to archive 2024-11-14T07:17:52,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:17:52,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9b9aea32f344499ca13956bafdb3d64e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9b9aea32f344499ca13956bafdb3d64e 2024-11-14T07:17:52,645 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/cd612fcec7a44fbfac5216d8f97ae207 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/cd612fcec7a44fbfac5216d8f97ae207 2024-11-14T07:17:52,646 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/74a5d8ede6e84aeebd9e8f31ad800054 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/74a5d8ede6e84aeebd9e8f31ad800054 2024-11-14T07:17:52,646 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/fc75405d2acb48358f127454b43a0e72 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/fc75405d2acb48358f127454b43a0e72 2024-11-14T07:17:52,647 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9eaf8a76d524462681e615b3fc30fe04 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/9eaf8a76d524462681e615b3fc30fe04 2024-11-14T07:17:52,654 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ca5d32e344e04171a98f3b8330a5b771 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ca5d32e344e04171a98f3b8330a5b771 2024-11-14T07:17:52,655 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f56d1a63f8e14d758859ec309ffeef20 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f56d1a63f8e14d758859ec309ffeef20 2024-11-14T07:17:52,656 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ae35b3bcb17403b97d903b77bfbddb3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ae35b3bcb17403b97d903b77bfbddb3 2024-11-14T07:17:52,657 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d2b914c40d9647149327c8dc252b439a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d2b914c40d9647149327c8dc252b439a 2024-11-14T07:17:52,658 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/2eaea9ae3fc34e478941985ee7cb5c83 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/2eaea9ae3fc34e478941985ee7cb5c83 2024-11-14T07:17:52,659 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ab252864b2c04276af660ec77c7e1a2f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ab252864b2c04276af660ec77c7e1a2f 2024-11-14T07:17:52,659 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/a0f8266a85894d5a96de5ef221deb968 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/a0f8266a85894d5a96de5ef221deb968 2024-11-14T07:17:52,660 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/374c152a1d174f8fa2374d915d15e0cc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/374c152a1d174f8fa2374d915d15e0cc 2024-11-14T07:17:52,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/8d21cfa824844babbba9392540f46d38 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/8d21cfa824844babbba9392540f46d38 2024-11-14T07:17:52,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d3a79775127e45debe62952d1442476b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d3a79775127e45debe62952d1442476b 2024-11-14T07:17:52,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d16e24e36bab423dbea55cd16deabd83 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/d16e24e36bab423dbea55cd16deabd83 2024-11-14T07:17:52,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/153a3c8bf3254981a12ae8b82ee42806 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/153a3c8bf3254981a12ae8b82ee42806 2024-11-14T07:17:52,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/7dd0f4aa4b5c49a4a53de8fd1218eff5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/7dd0f4aa4b5c49a4a53de8fd1218eff5 2024-11-14T07:17:52,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/acd23a269e494bb1a61808ba889de1c2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/acd23a269e494bb1a61808ba889de1c2 2024-11-14T07:17:52,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ac5e9813476d46299d8c96e4e3ece518 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/ac5e9813476d46299d8c96e4e3ece518 2024-11-14T07:17:52,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f0f2e1691e744946a01309b9102626c1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/f0f2e1691e744946a01309b9102626c1 2024-11-14T07:17:52,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/49b7cbe7be234c5b8971b17305dc7e3c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/49b7cbe7be234c5b8971b17305dc7e3c 2024-11-14T07:17:52,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/87ae98710b884ede8bb8fc5b6e1ad2ef to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/87ae98710b884ede8bb8fc5b6e1ad2ef 2024-11-14T07:17:52,669 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/6d5e485be59a4c6b80412f47a6910787 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/6d5e485be59a4c6b80412f47a6910787 2024-11-14T07:17:52,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/45b5386f844d434ab3362575320f4620, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/29602c8c5ccc4104ac622d28536a0e63, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/bb82f657ace34b16989a9e5ea5c3781a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/2798a6d90ee741648145f500bc3795e4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f600b4740a96424aace592b08f251a9e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46af8dd41b9a48418781bb2693aa466c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/90f1a33f416a4c04be3ae1de5b82a912, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/3e2ac6f4128d4a58853a4e9238f31e92, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e70a701faac14695ace397952284265c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/ca6f7bc1793848e9b4c36e431544c596, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/69cff8ecd4d848e1992a5102eee1625c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/14f54455ab6e4a8185262ded7804f7a2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/d40e2bc087b943dbb744439c29aa30c7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/0f5b428681384ab4ba160305361ec455, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/732a9fe42e89412b901b513a7031afd0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/345240fb30c948f2a43009d90cf348eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e14c8066172144b6a98c0e3e5187f9a8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/cbbada88161a46558965c440bff4cf9b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/5d248fa1a5b44ec79c356cee7a375db8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46d8afdd4887430a9a39ba5dc814fe8f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/42227423a18245898e87df8099531a96, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f23f7153b7104039a3b96ee81f08bd74] to archive 2024-11-14T07:17:52,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:17:52,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/45b5386f844d434ab3362575320f4620 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/45b5386f844d434ab3362575320f4620 2024-11-14T07:17:52,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/29602c8c5ccc4104ac622d28536a0e63 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/29602c8c5ccc4104ac622d28536a0e63 2024-11-14T07:17:52,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/bb82f657ace34b16989a9e5ea5c3781a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/bb82f657ace34b16989a9e5ea5c3781a 2024-11-14T07:17:52,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/2798a6d90ee741648145f500bc3795e4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/2798a6d90ee741648145f500bc3795e4 2024-11-14T07:17:52,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f600b4740a96424aace592b08f251a9e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f600b4740a96424aace592b08f251a9e 2024-11-14T07:17:52,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46af8dd41b9a48418781bb2693aa466c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46af8dd41b9a48418781bb2693aa466c 2024-11-14T07:17:52,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b8b4ec0b8b2d41e6b2de3aacc7d5ae7f 2024-11-14T07:17:52,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/90f1a33f416a4c04be3ae1de5b82a912 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/90f1a33f416a4c04be3ae1de5b82a912 2024-11-14T07:17:52,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/3e2ac6f4128d4a58853a4e9238f31e92 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/3e2ac6f4128d4a58853a4e9238f31e92 2024-11-14T07:17:52,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e70a701faac14695ace397952284265c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e70a701faac14695ace397952284265c 2024-11-14T07:17:52,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/ca6f7bc1793848e9b4c36e431544c596 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/ca6f7bc1793848e9b4c36e431544c596 2024-11-14T07:17:52,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/69cff8ecd4d848e1992a5102eee1625c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/69cff8ecd4d848e1992a5102eee1625c 2024-11-14T07:17:52,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/14f54455ab6e4a8185262ded7804f7a2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/14f54455ab6e4a8185262ded7804f7a2 2024-11-14T07:17:52,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/d40e2bc087b943dbb744439c29aa30c7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/d40e2bc087b943dbb744439c29aa30c7 2024-11-14T07:17:52,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/0f5b428681384ab4ba160305361ec455 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/0f5b428681384ab4ba160305361ec455 2024-11-14T07:17:52,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/732a9fe42e89412b901b513a7031afd0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/732a9fe42e89412b901b513a7031afd0 2024-11-14T07:17:52,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/345240fb30c948f2a43009d90cf348eb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/345240fb30c948f2a43009d90cf348eb 2024-11-14T07:17:52,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e14c8066172144b6a98c0e3e5187f9a8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/e14c8066172144b6a98c0e3e5187f9a8 2024-11-14T07:17:52,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/cbbada88161a46558965c440bff4cf9b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/cbbada88161a46558965c440bff4cf9b 2024-11-14T07:17:52,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/5d248fa1a5b44ec79c356cee7a375db8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/5d248fa1a5b44ec79c356cee7a375db8 2024-11-14T07:17:52,693 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46d8afdd4887430a9a39ba5dc814fe8f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/46d8afdd4887430a9a39ba5dc814fe8f 2024-11-14T07:17:52,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/42227423a18245898e87df8099531a96 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/42227423a18245898e87df8099531a96 2024-11-14T07:17:52,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f23f7153b7104039a3b96ee81f08bd74 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f23f7153b7104039a3b96ee81f08bd74 2024-11-14T07:17:53,763 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-14T07:17:53,763 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:17:53,763 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:53,764 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:53,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3153 rows 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1048 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3143 rows 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3151 rows 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3148 rows 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-11-14T07:17:53,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3157 rows 2024-11-14T07:17:53,764 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:17:53,764 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:17:53,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:53,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:53,765 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:17:53,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:17:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:17:53,769 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568673769"}]},"ts":"1731568673769"} 2024-11-14T07:17:53,770 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:17:53,770 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:17:53,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:17:53,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, UNASSIGN}] 2024-11-14T07:17:53,772 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, UNASSIGN 2024-11-14T07:17:53,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:53,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, UNASSIGN because future has completed 2024-11-14T07:17:53,774 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:17:53,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:17:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:17:53,926 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:53,926 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:17:53,926 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 17b20244019df3f0f04e41107ed48efc, disabling compactions & flushes 2024-11-14T07:17:53,926 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:53,926 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:53,926 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. after waiting 0 ms 2024-11-14T07:17:53,926 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:53,926 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 17b20244019df3f0f04e41107ed48efc 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T07:17:53,927 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=A 2024-11-14T07:17:53,927 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:53,927 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=B 2024-11-14T07:17:53,927 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:53,927 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17b20244019df3f0f04e41107ed48efc, store=C 2024-11-14T07:17:53,927 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:53,931 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140978c51c54f9408c9da6cf85b44312a9_17b20244019df3f0f04e41107ed48efc is 50, key is test_row_0/A:col10/1731568667412/Put/seqid=0 2024-11-14T07:17:53,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742249_1425 (size=12454) 2024-11-14T07:17:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:17:54,335 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:54,338 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140978c51c54f9408c9da6cf85b44312a9_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140978c51c54f9408c9da6cf85b44312a9_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:54,339 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/d6202eea11f847f399762757b833c98a, store: [table=TestAcidGuarantees family=A region=17b20244019df3f0f04e41107ed48efc] 2024-11-14T07:17:54,339 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/d6202eea11f847f399762757b833c98a is 175, key is test_row_0/A:col10/1731568667412/Put/seqid=0 2024-11-14T07:17:54,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742250_1426 (size=31255) 2024-11-14T07:17:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:17:54,743 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/d6202eea11f847f399762757b833c98a 2024-11-14T07:17:54,749 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/36f8e672deab4a01b49cc62f9e13fef3 is 50, key is test_row_0/B:col10/1731568667412/Put/seqid=0 2024-11-14T07:17:54,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742251_1427 (size=12301) 2024-11-14T07:17:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:17:55,152 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/36f8e672deab4a01b49cc62f9e13fef3 2024-11-14T07:17:55,158 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/8d27ecbb2e934e84b9b503907a50f915 is 50, key is test_row_0/C:col10/1731568667412/Put/seqid=0 2024-11-14T07:17:55,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742252_1428 (size=12301) 2024-11-14T07:17:55,562 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/8d27ecbb2e934e84b9b503907a50f915 2024-11-14T07:17:55,565 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/A/d6202eea11f847f399762757b833c98a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/d6202eea11f847f399762757b833c98a 2024-11-14T07:17:55,568 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/d6202eea11f847f399762757b833c98a, entries=150, sequenceid=413, filesize=30.5 K 2024-11-14T07:17:55,568 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/B/36f8e672deab4a01b49cc62f9e13fef3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/36f8e672deab4a01b49cc62f9e13fef3 2024-11-14T07:17:55,571 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/36f8e672deab4a01b49cc62f9e13fef3, entries=150, sequenceid=413, filesize=12.0 K 2024-11-14T07:17:55,572 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/.tmp/C/8d27ecbb2e934e84b9b503907a50f915 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/8d27ecbb2e934e84b9b503907a50f915 2024-11-14T07:17:55,575 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/8d27ecbb2e934e84b9b503907a50f915, entries=150, sequenceid=413, filesize=12.0 K 2024-11-14T07:17:55,575 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 17b20244019df3f0f04e41107ed48efc in 1649ms, sequenceid=413, compaction requested=true 2024-11-14T07:17:55,579 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/recovered.edits/416.seqid, newMaxSeqId=416, maxSeqId=4 2024-11-14T07:17:55,579 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. 2024-11-14T07:17:55,579 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 17b20244019df3f0f04e41107ed48efc: Waiting for close lock at 1731568673926Running coprocessor pre-close hooks at 1731568673926Disabling compacts and flushes for region at 1731568673926Disabling writes for close at 1731568673926Obtaining lock to block concurrent updates at 1731568673926Preparing flush snapshotting stores in 17b20244019df3f0f04e41107ed48efc at 1731568673926Finished memstore snapshotting TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731568673927 (+1 ms)Flushing stores of TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc. at 1731568673927Flushing 17b20244019df3f0f04e41107ed48efc/A: creating writer at 1731568673928 (+1 ms)Flushing mob file 17b20244019df3f0f04e41107ed48efc/A: appending metadata at 1731568673931 (+3 ms)Flushing mob file 17b20244019df3f0f04e41107ed48efc/A: closing flushed file at 1731568673931Flushing 17b20244019df3f0f04e41107ed48efc/A: appending metadata at 1731568674339 (+408 ms)Flushing 17b20244019df3f0f04e41107ed48efc/A: closing flushed file at 1731568674339Flushing 17b20244019df3f0f04e41107ed48efc/B: creating writer at 1731568674746 (+407 ms)Flushing 17b20244019df3f0f04e41107ed48efc/B: appending metadata at 1731568674748 (+2 ms)Flushing 17b20244019df3f0f04e41107ed48efc/B: closing flushed file at 1731568674748Flushing 17b20244019df3f0f04e41107ed48efc/C: creating writer at 1731568675155 (+407 ms)Flushing 17b20244019df3f0f04e41107ed48efc/C: appending metadata at 1731568675158 (+3 ms)Flushing 17b20244019df3f0f04e41107ed48efc/C: closing flushed file at 1731568675158Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4de3b30e: reopening flushed file at 1731568675565 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ec6e61e: reopening flushed file at 1731568675568 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@428249ca: reopening flushed file at 1731568675571 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 17b20244019df3f0f04e41107ed48efc in 1649ms, sequenceid=413, compaction requested=true at 1731568675575 (+4 ms)Writing region close event to WAL at 1731568675576 (+1 ms)Running coprocessor post-close hooks at 1731568675579 (+3 ms)Closed at 1731568675579 2024-11-14T07:17:55,581 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,581 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=17b20244019df3f0f04e41107ed48efc, regionState=CLOSED 2024-11-14T07:17:55,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:17:55,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T07:17:55,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 17b20244019df3f0f04e41107ed48efc, server=c7aa6e2790b3,33845,1731568550163 in 1.8090 sec 2024-11-14T07:17:55,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-14T07:17:55,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b20244019df3f0f04e41107ed48efc, UNASSIGN in 1.8120 sec 2024-11-14T07:17:55,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T07:17:55,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8150 sec 2024-11-14T07:17:55,588 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568675588"}]},"ts":"1731568675588"} 2024-11-14T07:17:55,589 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:17:55,589 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:17:55,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8250 sec 2024-11-14T07:17:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:17:55,913 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:55,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:17:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:55,915 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:17:55,915 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:55,917 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,919 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/recovered.edits] 2024-11-14T07:17:55,921 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/4be66c00f5c54678b04c30dd53deb064 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/4be66c00f5c54678b04c30dd53deb064 2024-11-14T07:17:55,922 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a12ef7a61b80422389c396d9f792e8d1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/a12ef7a61b80422389c396d9f792e8d1 2024-11-14T07:17:55,922 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/d6202eea11f847f399762757b833c98a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/A/d6202eea11f847f399762757b833c98a 2024-11-14T07:17:55,924 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ea69f693b934c4a95187870ad958d35 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/0ea69f693b934c4a95187870ad958d35 2024-11-14T07:17:55,925 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/36f8e672deab4a01b49cc62f9e13fef3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/36f8e672deab4a01b49cc62f9e13fef3 2024-11-14T07:17:55,927 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/bbea26c43bb14115a35260a6f363818a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/B/bbea26c43bb14115a35260a6f363818a 2024-11-14T07:17:55,929 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/8d27ecbb2e934e84b9b503907a50f915 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/8d27ecbb2e934e84b9b503907a50f915 2024-11-14T07:17:55,930 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b4b077882c974633b26d5317938303a5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/b4b077882c974633b26d5317938303a5 2024-11-14T07:17:55,931 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f895b7933dcb4749a6fb7c08de6e735f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/C/f895b7933dcb4749a6fb7c08de6e735f 2024-11-14T07:17:55,933 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/recovered.edits/416.seqid to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc/recovered.edits/416.seqid 2024-11-14T07:17:55,934 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,934 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:17:55,934 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:17:55,935 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T07:17:55,937 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111404791f511a3342549db61ae181f174b2_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111404791f511a3342549db61ae181f174b2_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,938 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140978c51c54f9408c9da6cf85b44312a9_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140978c51c54f9408c9da6cf85b44312a9_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,938 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142583a76e50d0495685f1f3fa4e0dccd4_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142583a76e50d0495685f1f3fa4e0dccd4_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,939 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143216535fc5ba495e822e3b8ff38b388c_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143216535fc5ba495e822e3b8ff38b388c_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,940 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111434c81f0584a641fc9e56665b59293761_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111434c81f0584a641fc9e56665b59293761_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,941 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111435a91c936a804024935888fc74e2b46b_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111435a91c936a804024935888fc74e2b46b_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,942 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114391635d5af7b440381633c53087e4f74_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114391635d5af7b440381633c53087e4f74_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,943 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144cf1248b4df24e4097b8d5e305f53644_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144cf1248b4df24e4097b8d5e305f53644_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,944 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114549e5a8f6d5a456f8fb2fb0e41e35fb5_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114549e5a8f6d5a456f8fb2fb0e41e35fb5_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,945 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114586bc08b48904941b83c00a3bed2c501_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114586bc08b48904941b83c00a3bed2c501_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,945 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114713ef52140eb433d9a468690ae298136_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114713ef52140eb433d9a468690ae298136_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,946 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147fe46c3daa494d109730b1a869ffdd6e_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147fe46c3daa494d109730b1a869ffdd6e_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,947 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486a00c4ba0c845028aa1cad0823a8025_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486a00c4ba0c845028aa1cad0823a8025_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,948 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148d4655985f5944e18af310ee8c70b9e7_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148d4655985f5944e18af310ee8c70b9e7_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,949 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114902498bf051c479d9885460cd77935a4_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114902498bf051c479d9885460cd77935a4_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,949 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae781d7dd6ed42a0b133e0cf90bb8bea_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae781d7dd6ed42a0b133e0cf90bb8bea_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,950 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ceb6dd3f86ee4fe59b406e247832c349_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ceb6dd3f86ee4fe59b406e247832c349_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,951 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d99f8a3d4db341debe29f59bbbdba48a_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d99f8a3d4db341debe29f59bbbdba48a_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,952 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e5ca082febd443ef927edc39525be4be_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e5ca082febd443ef927edc39525be4be_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,953 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fc3e13b4459342338afa19d399e0328b_17b20244019df3f0f04e41107ed48efc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fc3e13b4459342338afa19d399e0328b_17b20244019df3f0f04e41107ed48efc 2024-11-14T07:17:55,953 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:17:55,955 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:55,957 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:17:55,958 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:17:55,959 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:55,959 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:17:55,959 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731568675959"}]},"ts":"9223372036854775807"} 2024-11-14T07:17:55,961 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:17:55,961 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 17b20244019df3f0f04e41107ed48efc, NAME => 'TestAcidGuarantees,,1731568642175.17b20244019df3f0f04e41107ed48efc.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:17:55,961 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:17:55,961 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731568675961"}]},"ts":"9223372036854775807"} 2024-11-14T07:17:55,963 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:17:55,963 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:55,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-14T07:17:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:17:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:17:56,023 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:56,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 248), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=452 (was 452), ProcessCount=11 (was 11), AvailableMemoryMB=5545 (was 6150) 2024-11-14T07:17:56,038 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=452, ProcessCount=11, AvailableMemoryMB=5545 2024-11-14T07:17:56,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:17:56,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:17:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:17:56,041 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:17:56,042 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:56,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-14T07:17:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T07:17:56,042 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:17:56,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742253_1429 (size=1061) 2024-11-14T07:17:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T07:17:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T07:17:56,449 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:17:56,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742254_1430 (size=53) 2024-11-14T07:17:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T07:17:56,854 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:17:56,854 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6537a8f9e4f66b0d85507b1f78377e39, disabling compactions & flushes 2024-11-14T07:17:56,854 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:56,854 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:56,854 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. after waiting 0 ms 2024-11-14T07:17:56,854 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:56,854 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:56,854 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6537a8f9e4f66b0d85507b1f78377e39: Waiting for close lock at 1731568676854Disabling compacts and flushes for region at 1731568676854Disabling writes for close at 1731568676854Writing region close event to WAL at 1731568676854Closed at 1731568676854 2024-11-14T07:17:56,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:17:56,855 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731568676855"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731568676855"}]},"ts":"1731568676855"} 2024-11-14T07:17:56,856 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:17:56,857 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:17:56,857 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568676857"}]},"ts":"1731568676857"} 2024-11-14T07:17:56,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:17:56,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, ASSIGN}] 2024-11-14T07:17:56,859 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, ASSIGN 2024-11-14T07:17:56,860 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=false 2024-11-14T07:17:57,010 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6537a8f9e4f66b0d85507b1f78377e39, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:57,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, ASSIGN because future has completed 2024-11-14T07:17:57,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:17:57,166 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:57,166 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:17:57,167 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,167 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:17:57,167 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,167 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,168 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,169 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:57,169 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6537a8f9e4f66b0d85507b1f78377e39 columnFamilyName A 2024-11-14T07:17:57,169 DEBUG [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:57,169 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.HStore(327): Store=6537a8f9e4f66b0d85507b1f78377e39/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:57,169 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,170 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:57,170 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6537a8f9e4f66b0d85507b1f78377e39 columnFamilyName B 2024-11-14T07:17:57,170 DEBUG [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:57,171 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.HStore(327): Store=6537a8f9e4f66b0d85507b1f78377e39/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:57,171 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,172 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:17:57,172 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6537a8f9e4f66b0d85507b1f78377e39 columnFamilyName C 2024-11-14T07:17:57,172 DEBUG [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:17:57,172 INFO [StoreOpener-6537a8f9e4f66b0d85507b1f78377e39-1 {}] regionserver.HStore(327): Store=6537a8f9e4f66b0d85507b1f78377e39/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:17:57,172 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:57,172 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,173 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,173 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,174 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,174 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,174 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:17:57,175 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,176 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:17:57,176 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 6537a8f9e4f66b0d85507b1f78377e39; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73732861, jitterRate=0.0987052470445633}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:17:57,176 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:57,177 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 6537a8f9e4f66b0d85507b1f78377e39: Running coprocessor pre-open hook at 1731568677167Writing region info on filesystem at 1731568677167Initializing all the Stores at 1731568677168 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568677168Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568677168Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568677168Cleaning up temporary data from old regions at 1731568677174 (+6 ms)Running coprocessor post-open hooks at 1731568677176 (+2 ms)Region opened successfully at 1731568677177 (+1 ms) 2024-11-14T07:17:57,177 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., pid=118, masterSystemTime=1731568677164 2024-11-14T07:17:57,179 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:57,179 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:57,179 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6537a8f9e4f66b0d85507b1f78377e39, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:57,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:17:57,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T07:17:57,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 in 169 msec 2024-11-14T07:17:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T07:17:57,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-14T07:17:57,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, ASSIGN in 324 msec 2024-11-14T07:17:57,185 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:17:57,185 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568677185"}]},"ts":"1731568677185"} 2024-11-14T07:17:57,186 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:17:57,187 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:17:57,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-14T07:17:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T07:17:58,192 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:17:58,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,194 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f252ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,195 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540ea45f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,198 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:17:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,37073,1731568549456 2024-11-14T07:17:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@217db6f3 2024-11-14T07:17:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:17:58,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:17:58,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,201 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7838e7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,202 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47242d47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,204 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cecb768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b06009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,206 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c26afd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b78adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,208 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6212b914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0beb39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,211 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf8769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,212 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f6cf0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29799f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460d8d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,216 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5090286d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5606119b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,218 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64eca3a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,219 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49238596, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,221 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40df987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,222 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5c4335, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:17:58,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:17:58,223 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:17:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:17:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:17:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f10a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:17:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:17:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:17:58,224 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:17:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f1dd92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:17:58,225 DEBUG [Thread-1953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,225 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,225 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,225 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,225 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,225 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,226 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,228 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:17:58,228 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,227 DEBUG [Thread-1955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,227 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T07:17:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,230 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:17:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:17:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:17:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:17:58,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:17:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,234 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:17:58,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:17:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2] 2024-11-14T07:17:58,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:17:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:17:58,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:17:58,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:58,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:17:58,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:58,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:17:58,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:58,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/8de5db060b43465bb33e2b09f7da9538 is 50, key is test_row_0/A:col10/1731568678244/Put/seqid=0 2024-11-14T07:17:58,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742255_1431 (size=12001) 2024-11-14T07:17:58,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568738271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568738272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568738272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568738273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568738281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:17:58,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568738392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568738392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568738392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568738392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568738392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:17:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568738602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568738602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568738602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568738602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568738602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/8de5db060b43465bb33e2b09f7da9538 2024-11-14T07:17:58,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/110bc9ce30664870b9a2e614b3f86f13 is 50, key is test_row_0/B:col10/1731568678244/Put/seqid=0 2024-11-14T07:17:58,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742256_1432 (size=12001) 2024-11-14T07:17:58,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:58,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:58,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:58,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:17:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568738912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568738912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568738922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568738922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568738922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:59,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/110bc9ce30664870b9a2e614b3f86f13 2024-11-14T07:17:59,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/f044b3f0ecc446e285b573b9157d0623 is 50, key is test_row_0/C:col10/1731568678244/Put/seqid=0 2024-11-14T07:17:59,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742257_1433 (size=12001) 2024-11-14T07:17:59,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:59,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:59,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:59,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:17:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568739421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568739421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:59,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568739432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:59,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568739432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:59,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:17:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568739442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:17:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:17:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:17:59,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:17:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:17:59,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/f044b3f0ecc446e285b573b9157d0623 2024-11-14T07:17:59,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/8de5db060b43465bb33e2b09f7da9538 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/8de5db060b43465bb33e2b09f7da9538 2024-11-14T07:17:59,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/8de5db060b43465bb33e2b09f7da9538, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:17:59,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/110bc9ce30664870b9a2e614b3f86f13 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/110bc9ce30664870b9a2e614b3f86f13 2024-11-14T07:17:59,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/110bc9ce30664870b9a2e614b3f86f13, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:17:59,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/f044b3f0ecc446e285b573b9157d0623 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/f044b3f0ecc446e285b573b9157d0623 2024-11-14T07:17:59,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/f044b3f0ecc446e285b573b9157d0623, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:17:59,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6537a8f9e4f66b0d85507b1f78377e39 in 1313ms, sequenceid=13, compaction requested=false 2024-11-14T07:17:59,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:17:59,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T07:17:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:17:59,612 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:17:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:17:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:17:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:17:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:17:59,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/751e0ce35d534a9abbbe03eda45caeb5 is 50, key is test_row_0/A:col10/1731568678280/Put/seqid=0 2024-11-14T07:17:59,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742258_1434 (size=12001) 2024-11-14T07:18:00,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/751e0ce35d534a9abbbe03eda45caeb5 2024-11-14T07:18:00,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/77761feab281411f99f4ad1a3eafd56a is 50, key is test_row_0/B:col10/1731568678280/Put/seqid=0 2024-11-14T07:18:00,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742259_1435 (size=12001) 2024-11-14T07:18:00,034 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/77761feab281411f99f4ad1a3eafd56a 2024-11-14T07:18:00,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/10a18fc70dcb4cad9c850a1dd79faa76 is 50, key is test_row_0/C:col10/1731568678280/Put/seqid=0 2024-11-14T07:18:00,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742260_1436 (size=12001) 2024-11-14T07:18:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:18:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:00,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:00,447 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/10a18fc70dcb4cad9c850a1dd79faa76 2024-11-14T07:18:00,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/751e0ce35d534a9abbbe03eda45caeb5 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/751e0ce35d534a9abbbe03eda45caeb5 2024-11-14T07:18:00,458 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/751e0ce35d534a9abbbe03eda45caeb5, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T07:18:00,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/77761feab281411f99f4ad1a3eafd56a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/77761feab281411f99f4ad1a3eafd56a 2024-11-14T07:18:00,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/77761feab281411f99f4ad1a3eafd56a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T07:18:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/10a18fc70dcb4cad9c850a1dd79faa76 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/10a18fc70dcb4cad9c850a1dd79faa76 2024-11-14T07:18:00,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568740455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568740463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568740463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,467 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/10a18fc70dcb4cad9c850a1dd79faa76, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T07:18:00,468 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 6537a8f9e4f66b0d85507b1f78377e39 in 856ms, sequenceid=33, compaction requested=false 2024-11-14T07:18:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T07:18:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T07:18:00,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T07:18:00,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2350 sec 2024-11-14T07:18:00,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.2420 sec 2024-11-14T07:18:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:00,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T07:18:00,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:00,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:00,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:00,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:00,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:00,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:00,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/d8f9a683947b4f7db177149110fc52c8 is 50, key is test_row_0/A:col10/1731568680462/Put/seqid=0 2024-11-14T07:18:00,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742261_1437 (size=14341) 2024-11-14T07:18:00,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/d8f9a683947b4f7db177149110fc52c8 2024-11-14T07:18:00,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/6ce27dad64744b34ba70d3044f22217f is 50, key is test_row_0/B:col10/1731568680462/Put/seqid=0 2024-11-14T07:18:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742262_1438 (size=12001) 2024-11-14T07:18:00,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/6ce27dad64744b34ba70d3044f22217f 2024-11-14T07:18:00,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/7f00f9ad8b2a454d8c92e98e4831b25b is 50, key is test_row_0/C:col10/1731568680462/Put/seqid=0 2024-11-14T07:18:00,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568740554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568740555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742263_1439 (size=12001) 2024-11-14T07:18:00,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/7f00f9ad8b2a454d8c92e98e4831b25b 2024-11-14T07:18:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568740572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568740572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568740573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/d8f9a683947b4f7db177149110fc52c8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/d8f9a683947b4f7db177149110fc52c8 2024-11-14T07:18:00,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/d8f9a683947b4f7db177149110fc52c8, entries=200, sequenceid=47, filesize=14.0 K 2024-11-14T07:18:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/6ce27dad64744b34ba70d3044f22217f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6ce27dad64744b34ba70d3044f22217f 2024-11-14T07:18:00,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6ce27dad64744b34ba70d3044f22217f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T07:18:00,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/7f00f9ad8b2a454d8c92e98e4831b25b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/7f00f9ad8b2a454d8c92e98e4831b25b 2024-11-14T07:18:00,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/7f00f9ad8b2a454d8c92e98e4831b25b, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T07:18:00,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6537a8f9e4f66b0d85507b1f78377e39 in 161ms, sequenceid=47, compaction requested=true 2024-11-14T07:18:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:00,633 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:00,633 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:00,634 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:00,634 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:00,634 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:00,634 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/110bc9ce30664870b9a2e614b3f86f13, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/77761feab281411f99f4ad1a3eafd56a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6ce27dad64744b34ba70d3044f22217f] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.2 K 2024-11-14T07:18:00,634 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:00,634 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:00,634 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:00,634 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/8de5db060b43465bb33e2b09f7da9538, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/751e0ce35d534a9abbbe03eda45caeb5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/d8f9a683947b4f7db177149110fc52c8] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=37.4 K 2024-11-14T07:18:00,634 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 110bc9ce30664870b9a2e614b3f86f13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731568678244 2024-11-14T07:18:00,634 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8de5db060b43465bb33e2b09f7da9538, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731568678244 2024-11-14T07:18:00,635 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 77761feab281411f99f4ad1a3eafd56a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731568678257 2024-11-14T07:18:00,635 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce27dad64744b34ba70d3044f22217f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568680462 2024-11-14T07:18:00,635 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 751e0ce35d534a9abbbe03eda45caeb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731568678257 2024-11-14T07:18:00,636 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f9a683947b4f7db177149110fc52c8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568680449 2024-11-14T07:18:00,646 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:00,646 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:00,646 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c39c91e74ea045bda6feafc8d4c2ec3a is 50, key is test_row_0/B:col10/1731568680462/Put/seqid=0 2024-11-14T07:18:00,647 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/e0f5322411654ed4836cda696a2ff091 is 50, key is test_row_0/A:col10/1731568680462/Put/seqid=0 2024-11-14T07:18:00,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742265_1441 (size=12104) 2024-11-14T07:18:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:00,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:00,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:00,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:00,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:00,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:00,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:00,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:00,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742264_1440 (size=12104) 2024-11-14T07:18:00,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/95dcf55f924d430fbb3aec0071f6202e is 50, key is test_row_0/A:col10/1731568680685/Put/seqid=0 2024-11-14T07:18:00,701 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c39c91e74ea045bda6feafc8d4c2ec3a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c39c91e74ea045bda6feafc8d4c2ec3a 2024-11-14T07:18:00,706 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into c39c91e74ea045bda6feafc8d4c2ec3a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:00,706 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:00,706 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=13, startTime=1731568680633; duration=0sec 2024-11-14T07:18:00,706 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:00,706 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:00,706 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:00,708 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:00,708 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:00,714 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:00,714 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/f044b3f0ecc446e285b573b9157d0623, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/10a18fc70dcb4cad9c850a1dd79faa76, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/7f00f9ad8b2a454d8c92e98e4831b25b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.2 K 2024-11-14T07:18:00,716 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f044b3f0ecc446e285b573b9157d0623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731568678244 2024-11-14T07:18:00,717 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a18fc70dcb4cad9c850a1dd79faa76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731568678257 2024-11-14T07:18:00,717 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f00f9ad8b2a454d8c92e98e4831b25b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568680462 2024-11-14T07:18:00,726 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:00,726 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c457368be5284d7fada9a32d4baa110b is 50, key is test_row_0/C:col10/1731568680462/Put/seqid=0 2024-11-14T07:18:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742266_1442 (size=12001) 2024-11-14T07:18:00,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568740742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742267_1443 (size=12104) 2024-11-14T07:18:00,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568740752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,763 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c457368be5284d7fada9a32d4baa110b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c457368be5284d7fada9a32d4baa110b 2024-11-14T07:18:00,768 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into c457368be5284d7fada9a32d4baa110b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:00,768 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:00,768 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=13, startTime=1731568680634; duration=0sec 2024-11-14T07:18:00,768 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:00,768 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:00,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568740791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568740792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568740792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568740861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:00,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568740871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:00,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,077 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/e0f5322411654ed4836cda696a2ff091 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e0f5322411654ed4836cda696a2ff091 2024-11-14T07:18:01,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568741071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,084 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into e0f5322411654ed4836cda696a2ff091(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:01,084 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:01,084 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=13, startTime=1731568680633; duration=0sec 2024-11-14T07:18:01,085 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:01,085 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:01,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568741092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568741102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568741102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568741102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,118 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:18:01,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/95dcf55f924d430fbb3aec0071f6202e 2024-11-14T07:18:01,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/caeefa8f89eb49b8bc2b707a623c686c is 50, key is test_row_0/B:col10/1731568680685/Put/seqid=0 2024-11-14T07:18:01,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742268_1444 (size=12001) 2024-11-14T07:18:01,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568741391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568741402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/caeefa8f89eb49b8bc2b707a623c686c 2024-11-14T07:18:01,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/cb4381c94dea4df1af8c572786bd9977 is 50, key is test_row_0/C:col10/1731568680685/Put/seqid=0 2024-11-14T07:18:01,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742269_1445 (size=12001) 2024-11-14T07:18:01,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/cb4381c94dea4df1af8c572786bd9977 2024-11-14T07:18:01,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/95dcf55f924d430fbb3aec0071f6202e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/95dcf55f924d430fbb3aec0071f6202e 2024-11-14T07:18:01,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/95dcf55f924d430fbb3aec0071f6202e, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T07:18:01,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/caeefa8f89eb49b8bc2b707a623c686c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/caeefa8f89eb49b8bc2b707a623c686c 2024-11-14T07:18:01,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/caeefa8f89eb49b8bc2b707a623c686c, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T07:18:01,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/cb4381c94dea4df1af8c572786bd9977 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cb4381c94dea4df1af8c572786bd9977 2024-11-14T07:18:01,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cb4381c94dea4df1af8c572786bd9977, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T07:18:01,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6537a8f9e4f66b0d85507b1f78377e39 in 923ms, sequenceid=66, compaction requested=false 2024-11-14T07:18:01,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:01,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T07:18:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:01,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/7e96cfbc666d43f584987c53d7b1a0ce is 50, key is test_row_0/A:col10/1731568681612/Put/seqid=0 2024-11-14T07:18:01,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742270_1446 (size=14341) 2024-11-14T07:18:01,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568741657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568741658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568741665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568741772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568741772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568741781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568741911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568741924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568741991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568741992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568741992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/7e96cfbc666d43f584987c53d7b1a0ce 2024-11-14T07:18:02,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/6a822112811d49c2963f152fa151ba7d is 50, key is test_row_0/B:col10/1731568681612/Put/seqid=0 2024-11-14T07:18:02,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742271_1447 (size=12001) 2024-11-14T07:18:02,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568742302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568742302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568742312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:18:02,412 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:02,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T07:18:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T07:18:02,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:02,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:02,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:02,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/6a822112811d49c2963f152fa151ba7d 2024-11-14T07:18:02,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/a1308205ca784990be61bf5324e6aebe is 50, key is test_row_0/C:col10/1731568681612/Put/seqid=0 2024-11-14T07:18:02,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742272_1448 (size=12001) 2024-11-14T07:18:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T07:18:02,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T07:18:02,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T07:18:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T07:18:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568742812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568742812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568742832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:02,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T07:18:02,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:02,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:02,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/a1308205ca784990be61bf5324e6aebe 2024-11-14T07:18:02,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/7e96cfbc666d43f584987c53d7b1a0ce as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7e96cfbc666d43f584987c53d7b1a0ce 2024-11-14T07:18:02,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7e96cfbc666d43f584987c53d7b1a0ce, entries=200, sequenceid=83, filesize=14.0 K 2024-11-14T07:18:02,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/6a822112811d49c2963f152fa151ba7d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6a822112811d49c2963f152fa151ba7d 2024-11-14T07:18:02,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6a822112811d49c2963f152fa151ba7d, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T07:18:02,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/a1308205ca784990be61bf5324e6aebe as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a1308205ca784990be61bf5324e6aebe 2024-11-14T07:18:02,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a1308205ca784990be61bf5324e6aebe, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T07:18:02,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6537a8f9e4f66b0d85507b1f78377e39 in 1309ms, sequenceid=83, compaction requested=true 2024-11-14T07:18:02,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:02,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:02,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:02,922 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:02,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:02,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:02,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:02,922 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:02,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:02,924 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:02,924 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:02,924 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,924 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e0f5322411654ed4836cda696a2ff091, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/95dcf55f924d430fbb3aec0071f6202e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7e96cfbc666d43f584987c53d7b1a0ce] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=37.5 K 2024-11-14T07:18:02,925 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f5322411654ed4836cda696a2ff091, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568680462 2024-11-14T07:18:02,926 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:02,926 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:02,926 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,926 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c39c91e74ea045bda6feafc8d4c2ec3a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/caeefa8f89eb49b8bc2b707a623c686c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6a822112811d49c2963f152fa151ba7d] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.3 K 2024-11-14T07:18:02,926 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c39c91e74ea045bda6feafc8d4c2ec3a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568680462 2024-11-14T07:18:02,926 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95dcf55f924d430fbb3aec0071f6202e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731568680549 2024-11-14T07:18:02,927 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting caeefa8f89eb49b8bc2b707a623c686c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731568680549 2024-11-14T07:18:02,927 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e96cfbc666d43f584987c53d7b1a0ce, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731568680727 2024-11-14T07:18:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:02,927 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a822112811d49c2963f152fa151ba7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731568680727 2024-11-14T07:18:02,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:02,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:02,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:02,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:02,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:02,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:02,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:02,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/03792df506db48abbe113432779293ee is 50, key is test_row_0/A:col10/1731568681657/Put/seqid=0 2024-11-14T07:18:02,939 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:02,939 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/11f33930ae054d2fae1d0ab870dc2f18 is 50, key is test_row_0/A:col10/1731568681612/Put/seqid=0 2024-11-14T07:18:02,942 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:02,942 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/2408815748d1499a9d050d9bef0f34ac is 50, key is test_row_0/B:col10/1731568681612/Put/seqid=0 2024-11-14T07:18:02,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742273_1449 (size=14341) 2024-11-14T07:18:02,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/03792df506db48abbe113432779293ee 2024-11-14T07:18:02,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8b8267aacd7a43a28171a6a6a77f58cb is 50, key is test_row_0/B:col10/1731568681657/Put/seqid=0 2024-11-14T07:18:02,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742275_1451 (size=12207) 2024-11-14T07:18:02,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742274_1450 (size=12207) 2024-11-14T07:18:02,960 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/2408815748d1499a9d050d9bef0f34ac as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2408815748d1499a9d050d9bef0f34ac 2024-11-14T07:18:02,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742276_1452 (size=12001) 2024-11-14T07:18:02,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8b8267aacd7a43a28171a6a6a77f58cb 2024-11-14T07:18:02,964 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into 2408815748d1499a9d050d9bef0f34ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:02,964 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:02,964 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=13, startTime=1731568682922; duration=0sec 2024-11-14T07:18:02,964 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:02,964 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:02,964 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:02,966 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:02,966 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:02,966 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:02,966 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c457368be5284d7fada9a32d4baa110b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cb4381c94dea4df1af8c572786bd9977, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a1308205ca784990be61bf5324e6aebe] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.3 K 2024-11-14T07:18:02,967 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c457368be5284d7fada9a32d4baa110b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731568680462 2024-11-14T07:18:02,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/22a666669e4b4bd19c9ef1dacf4a6356 is 50, key is test_row_0/C:col10/1731568681657/Put/seqid=0 2024-11-14T07:18:02,968 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cb4381c94dea4df1af8c572786bd9977, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731568680549 2024-11-14T07:18:02,969 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a1308205ca784990be61bf5324e6aebe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731568680727 2024-11-14T07:18:02,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742277_1453 (size=12001) 2024-11-14T07:18:02,975 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:02,975 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/cd5d0d0f002b4f3f999ae28544b7a044 is 50, key is test_row_0/C:col10/1731568681612/Put/seqid=0 2024-11-14T07:18:02,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/22a666669e4b4bd19c9ef1dacf4a6356 2024-11-14T07:18:02,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/03792df506db48abbe113432779293ee as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/03792df506db48abbe113432779293ee 2024-11-14T07:18:02,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/03792df506db48abbe113432779293ee, entries=200, sequenceid=102, filesize=14.0 K 2024-11-14T07:18:02,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8b8267aacd7a43a28171a6a6a77f58cb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8b8267aacd7a43a28171a6a6a77f58cb 2024-11-14T07:18:02,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8b8267aacd7a43a28171a6a6a77f58cb, entries=150, sequenceid=102, filesize=11.7 K 2024-11-14T07:18:02,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/22a666669e4b4bd19c9ef1dacf4a6356 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/22a666669e4b4bd19c9ef1dacf4a6356 2024-11-14T07:18:02,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/22a666669e4b4bd19c9ef1dacf4a6356, entries=150, sequenceid=102, filesize=11.7 K 2024-11-14T07:18:02,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=60.38 KB/61830 for 6537a8f9e4f66b0d85507b1f78377e39 in 64ms, sequenceid=102, compaction requested=false 2024-11-14T07:18:02,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:02,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:18:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:02,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742278_1454 (size=12207) 2024-11-14T07:18:03,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/eaacde2458494e1aa43ddc751751ef7e is 50, key is test_row_0/A:col10/1731568682979/Put/seqid=0 2024-11-14T07:18:03,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742279_1455 (size=14341) 2024-11-14T07:18:03,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/eaacde2458494e1aa43ddc751751ef7e 2024-11-14T07:18:03,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/bf4c06c1f9f2415f9fc998b98735963b is 50, key is test_row_0/B:col10/1731568682979/Put/seqid=0 2024-11-14T07:18:03,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T07:18:03,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:03,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742280_1456 (size=12001) 2024-11-14T07:18:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T07:18:03,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568743073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568743089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T07:18:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568743191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568743202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T07:18:03,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:03,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:03,362 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/11f33930ae054d2fae1d0ab870dc2f18 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/11f33930ae054d2fae1d0ab870dc2f18 2024-11-14T07:18:03,365 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into 11f33930ae054d2fae1d0ab870dc2f18(size=11.9 K), total size for store is 25.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:03,365 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:03,365 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=13, startTime=1731568682922; duration=0sec 2024-11-14T07:18:03,365 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:03,365 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:03,403 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/cd5d0d0f002b4f3f999ae28544b7a044 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd5d0d0f002b4f3f999ae28544b7a044 2024-11-14T07:18:03,407 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into cd5d0d0f002b4f3f999ae28544b7a044(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:03,407 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:03,407 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=13, startTime=1731568682922; duration=0sec 2024-11-14T07:18:03,407 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:03,407 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:03,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568743411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568743412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/bf4c06c1f9f2415f9fc998b98735963b 2024-11-14T07:18:03,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/400f6310f6e24cda9e67fd5e3f9e1a73 is 50, key is test_row_0/C:col10/1731568682979/Put/seqid=0 2024-11-14T07:18:03,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742281_1457 (size=12001) 2024-11-14T07:18:03,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/400f6310f6e24cda9e67fd5e3f9e1a73 2024-11-14T07:18:03,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/eaacde2458494e1aa43ddc751751ef7e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/eaacde2458494e1aa43ddc751751ef7e 2024-11-14T07:18:03,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/eaacde2458494e1aa43ddc751751ef7e, entries=200, sequenceid=116, filesize=14.0 K 2024-11-14T07:18:03,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/bf4c06c1f9f2415f9fc998b98735963b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/bf4c06c1f9f2415f9fc998b98735963b 2024-11-14T07:18:03,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/bf4c06c1f9f2415f9fc998b98735963b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T07:18:03,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/400f6310f6e24cda9e67fd5e3f9e1a73 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/400f6310f6e24cda9e67fd5e3f9e1a73 2024-11-14T07:18:03,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/400f6310f6e24cda9e67fd5e3f9e1a73, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T07:18:03,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6537a8f9e4f66b0d85507b1f78377e39 in 474ms, sequenceid=116, compaction requested=true 2024-11-14T07:18:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:03,469 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:03,469 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:03,470 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:03,470 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:03,470 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:03,470 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:03,470 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,470 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,470 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/11f33930ae054d2fae1d0ab870dc2f18, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/03792df506db48abbe113432779293ee, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/eaacde2458494e1aa43ddc751751ef7e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=39.9 K 2024-11-14T07:18:03,470 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2408815748d1499a9d050d9bef0f34ac, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8b8267aacd7a43a28171a6a6a77f58cb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/bf4c06c1f9f2415f9fc998b98735963b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.4 K 2024-11-14T07:18:03,470 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2408815748d1499a9d050d9bef0f34ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731568680727 2024-11-14T07:18:03,470 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11f33930ae054d2fae1d0ab870dc2f18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731568680727 2024-11-14T07:18:03,471 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b8267aacd7a43a28171a6a6a77f58cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731568681649 2024-11-14T07:18:03,471 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03792df506db48abbe113432779293ee, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731568681649 2024-11-14T07:18:03,471 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaacde2458494e1aa43ddc751751ef7e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731568682946 2024-11-14T07:18:03,471 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4c06c1f9f2415f9fc998b98735963b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731568682946 2024-11-14T07:18:03,478 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:03,479 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/a18a055e1fff483d8cba8ca84913817e is 50, key is test_row_0/B:col10/1731568682979/Put/seqid=0 2024-11-14T07:18:03,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T07:18:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,482 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:03,482 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:03,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:03,483 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4f01804b54ad480dae83149d242a5e93 is 50, key is test_row_0/A:col10/1731568682979/Put/seqid=0 2024-11-14T07:18:03,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/2b3d4d1b214646b18b485f97d12dc753 is 50, key is test_row_0/A:col10/1731568683070/Put/seqid=0 2024-11-14T07:18:03,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742282_1458 (size=12309) 2024-11-14T07:18:03,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742283_1459 (size=12309) 2024-11-14T07:18:03,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742284_1460 (size=12151) 2024-11-14T07:18:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T07:18:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:03,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568743788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568743794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568743821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,827 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568743831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,839 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568743852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,863 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,904 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/a18a055e1fff483d8cba8ca84913817e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a18a055e1fff483d8cba8ca84913817e 2024-11-14T07:18:03,907 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into a18a055e1fff483d8cba8ca84913817e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:03,907 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:03,907 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=13, startTime=1731568683469; duration=0sec 2024-11-14T07:18:03,907 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:03,908 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:03,908 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:03,909 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:03,909 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:03,909 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:03,910 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd5d0d0f002b4f3f999ae28544b7a044, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/22a666669e4b4bd19c9ef1dacf4a6356, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/400f6310f6e24cda9e67fd5e3f9e1a73] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.4 K 2024-11-14T07:18:03,910 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cd5d0d0f002b4f3f999ae28544b7a044, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731568680727 2024-11-14T07:18:03,910 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a666669e4b4bd19c9ef1dacf4a6356, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731568681649 2024-11-14T07:18:03,910 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 400f6310f6e24cda9e67fd5e3f9e1a73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731568682946 2024-11-14T07:18:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568743901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,918 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/2b3d4d1b214646b18b485f97d12dc753 2024-11-14T07:18:03,921 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:03,921 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/5ac16b0fc6c84e6bbf0a0eb1bc76f383 is 50, key is test_row_0/C:col10/1731568682979/Put/seqid=0 2024-11-14T07:18:03,921 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4f01804b54ad480dae83149d242a5e93 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4f01804b54ad480dae83149d242a5e93 2024-11-14T07:18:03,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568743912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8714205d3a544b9cb808a08323bc5108 is 50, key is test_row_0/B:col10/1731568683070/Put/seqid=0 2024-11-14T07:18:03,931 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into 4f01804b54ad480dae83149d242a5e93(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:03,931 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:03,931 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=13, startTime=1731568683469; duration=0sec 2024-11-14T07:18:03,932 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:03,932 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:03,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742285_1461 (size=12309) 2024-11-14T07:18:03,941 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/5ac16b0fc6c84e6bbf0a0eb1bc76f383 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5ac16b0fc6c84e6bbf0a0eb1bc76f383 2024-11-14T07:18:03,945 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into 5ac16b0fc6c84e6bbf0a0eb1bc76f383(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:03,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:03,945 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=13, startTime=1731568683469; duration=0sec 2024-11-14T07:18:03,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:03,945 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:03,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742286_1462 (size=12151) 2024-11-14T07:18:04,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568744121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:04,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568744131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:04,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8714205d3a544b9cb808a08323bc5108 2024-11-14T07:18:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/a3c8b6dcf7d24aee977f074dbb29c04f is 50, key is test_row_0/C:col10/1731568683070/Put/seqid=0 2024-11-14T07:18:04,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742287_1463 (size=12151) 2024-11-14T07:18:04,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568744432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:04,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:04,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568744442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T07:18:04,762 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/a3c8b6dcf7d24aee977f074dbb29c04f 2024-11-14T07:18:04,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/2b3d4d1b214646b18b485f97d12dc753 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/2b3d4d1b214646b18b485f97d12dc753 2024-11-14T07:18:04,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/2b3d4d1b214646b18b485f97d12dc753, entries=150, sequenceid=137, filesize=11.9 K 2024-11-14T07:18:04,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8714205d3a544b9cb808a08323bc5108 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8714205d3a544b9cb808a08323bc5108 2024-11-14T07:18:04,773 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8714205d3a544b9cb808a08323bc5108, entries=150, sequenceid=137, filesize=11.9 K 2024-11-14T07:18:04,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/a3c8b6dcf7d24aee977f074dbb29c04f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a3c8b6dcf7d24aee977f074dbb29c04f 2024-11-14T07:18:04,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a3c8b6dcf7d24aee977f074dbb29c04f, entries=150, sequenceid=137, filesize=11.9 K 2024-11-14T07:18:04,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6537a8f9e4f66b0d85507b1f78377e39 in 1300ms, sequenceid=137, compaction requested=false 2024-11-14T07:18:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T07:18:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T07:18:04,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T07:18:04,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-11-14T07:18:04,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.3730 sec 2024-11-14T07:18:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:04,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T07:18:04,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:04,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:04,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:04,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:04,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:04,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:04,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/bcad04bec9c3479cb2a795a48e8cacbe is 50, key is test_row_0/A:col10/1731568683787/Put/seqid=0 2024-11-14T07:18:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742288_1464 (size=14541) 2024-11-14T07:18:04,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/bcad04bec9c3479cb2a795a48e8cacbe 2024-11-14T07:18:04,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8d87b662d8cc4f8eaeaf93bf22166d46 is 50, key is test_row_0/B:col10/1731568683787/Put/seqid=0 2024-11-14T07:18:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742289_1465 (size=12151) 2024-11-14T07:18:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568745014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568745015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568745132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568745132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568745351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568745351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8d87b662d8cc4f8eaeaf93bf22166d46 2024-11-14T07:18:05,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/44628a7280f64e58955b05d67c8172c2 is 50, key is test_row_0/C:col10/1731568683787/Put/seqid=0 2024-11-14T07:18:05,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742290_1466 (size=12151) 2024-11-14T07:18:05,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568745672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568745673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/44628a7280f64e58955b05d67c8172c2 2024-11-14T07:18:05,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/bcad04bec9c3479cb2a795a48e8cacbe as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/bcad04bec9c3479cb2a795a48e8cacbe 2024-11-14T07:18:05,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/bcad04bec9c3479cb2a795a48e8cacbe, entries=200, sequenceid=154, filesize=14.2 K 2024-11-14T07:18:05,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8d87b662d8cc4f8eaeaf93bf22166d46 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8d87b662d8cc4f8eaeaf93bf22166d46 2024-11-14T07:18:05,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8d87b662d8cc4f8eaeaf93bf22166d46, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T07:18:05,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/44628a7280f64e58955b05d67c8172c2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/44628a7280f64e58955b05d67c8172c2 2024-11-14T07:18:05,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/44628a7280f64e58955b05d67c8172c2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T07:18:05,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6537a8f9e4f66b0d85507b1f78377e39 in 872ms, sequenceid=154, compaction requested=true 2024-11-14T07:18:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:05,831 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:05,831 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:05,832 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:05,832 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:05,832 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:05,832 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a18a055e1fff483d8cba8ca84913817e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8714205d3a544b9cb808a08323bc5108, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8d87b662d8cc4f8eaeaf93bf22166d46] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.8 K 2024-11-14T07:18:05,832 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:05,833 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:05,833 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:05,833 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a18a055e1fff483d8cba8ca84913817e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731568682946 2024-11-14T07:18:05,833 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4f01804b54ad480dae83149d242a5e93, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/2b3d4d1b214646b18b485f97d12dc753, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/bcad04bec9c3479cb2a795a48e8cacbe] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=38.1 K 2024-11-14T07:18:05,833 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8714205d3a544b9cb808a08323bc5108, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731568683068 2024-11-14T07:18:05,833 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f01804b54ad480dae83149d242a5e93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731568682946 2024-11-14T07:18:05,833 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3d4d1b214646b18b485f97d12dc753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731568683068 2024-11-14T07:18:05,833 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d87b662d8cc4f8eaeaf93bf22166d46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731568683787 2024-11-14T07:18:05,834 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcad04bec9c3479cb2a795a48e8cacbe, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731568683786 2024-11-14T07:18:05,842 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:05,843 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/c9b088d3c13247a08dfa654b1ab2f39e is 50, key is test_row_0/A:col10/1731568683787/Put/seqid=0 2024-11-14T07:18:05,846 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:05,846 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c5f8a7f288f3465b8d3be83ab1c50233 is 50, key is test_row_0/B:col10/1731568683787/Put/seqid=0 2024-11-14T07:18:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:05,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:05,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742291_1467 (size=12561) 2024-11-14T07:18:05,878 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/c9b088d3c13247a08dfa654b1ab2f39e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c9b088d3c13247a08dfa654b1ab2f39e 2024-11-14T07:18:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742292_1468 (size=12561) 2024-11-14T07:18:05,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4e93d18e16524905948d348eca5664eb is 50, key is test_row_0/A:col10/1731568685013/Put/seqid=0 2024-11-14T07:18:05,886 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into c9b088d3c13247a08dfa654b1ab2f39e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:05,886 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:05,886 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=13, startTime=1731568685831; duration=0sec 2024-11-14T07:18:05,886 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:05,886 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:05,886 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:05,887 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:05,887 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:05,887 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:05,887 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5ac16b0fc6c84e6bbf0a0eb1bc76f383, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a3c8b6dcf7d24aee977f074dbb29c04f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/44628a7280f64e58955b05d67c8172c2] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=35.8 K 2024-11-14T07:18:05,887 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac16b0fc6c84e6bbf0a0eb1bc76f383, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731568682946 2024-11-14T07:18:05,888 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3c8b6dcf7d24aee977f074dbb29c04f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731568683068 2024-11-14T07:18:05,888 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44628a7280f64e58955b05d67c8172c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731568683787 2024-11-14T07:18:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742293_1469 (size=14541) 2024-11-14T07:18:05,897 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:05,897 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ed1a1a67a75d4fd8b9746ff10d6ecbba is 50, key is test_row_0/C:col10/1731568683787/Put/seqid=0 2024-11-14T07:18:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568745897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568745901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:05,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742294_1470 (size=12561) 2024-11-14T07:18:05,913 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ed1a1a67a75d4fd8b9746ff10d6ecbba as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ed1a1a67a75d4fd8b9746ff10d6ecbba 2024-11-14T07:18:05,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568745902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:05,916 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into ed1a1a67a75d4fd8b9746ff10d6ecbba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:05,916 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:05,916 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=13, startTime=1731568685831; duration=0sec 2024-11-14T07:18:05,916 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:05,916 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568746011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568746011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568746022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568746182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568746192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568746221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568746222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568746242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,285 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c5f8a7f288f3465b8d3be83ab1c50233 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c5f8a7f288f3465b8d3be83ab1c50233 2024-11-14T07:18:06,289 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into c5f8a7f288f3465b8d3be83ab1c50233(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:06,289 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:06,289 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=13, startTime=1731568685831; duration=0sec 2024-11-14T07:18:06,289 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:06,289 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:06,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4e93d18e16524905948d348eca5664eb 2024-11-14T07:18:06,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e2e88447606a4fa49c678a5c85679bea is 50, key is test_row_0/B:col10/1731568685013/Put/seqid=0 2024-11-14T07:18:06,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742295_1471 (size=12151) 2024-11-14T07:18:06,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568746532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568746542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568746561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T07:18:06,582 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:06,582 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T07:18:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:18:06,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:06,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:06,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:18:06,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e2e88447606a4fa49c678a5c85679bea 2024-11-14T07:18:06,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/d49cef310bc840e9a4f50a677103b49e is 50, key is test_row_0/C:col10/1731568685013/Put/seqid=0 2024-11-14T07:18:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742296_1472 (size=12151) 2024-11-14T07:18:06,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/d49cef310bc840e9a4f50a677103b49e 2024-11-14T07:18:06,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4e93d18e16524905948d348eca5664eb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4e93d18e16524905948d348eca5664eb 2024-11-14T07:18:06,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4e93d18e16524905948d348eca5664eb, entries=200, sequenceid=173, filesize=14.2 K 2024-11-14T07:18:06,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T07:18:06,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:06,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:06,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:06,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:06,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e2e88447606a4fa49c678a5c85679bea as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2e88447606a4fa49c678a5c85679bea 2024-11-14T07:18:06,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2e88447606a4fa49c678a5c85679bea, entries=150, sequenceid=173, filesize=11.9 K 2024-11-14T07:18:06,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/d49cef310bc840e9a4f50a677103b49e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d49cef310bc840e9a4f50a677103b49e 2024-11-14T07:18:06,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d49cef310bc840e9a4f50a677103b49e, entries=150, sequenceid=173, filesize=11.9 K 2024-11-14T07:18:06,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6537a8f9e4f66b0d85507b1f78377e39 in 888ms, sequenceid=173, compaction requested=false 2024-11-14T07:18:06,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:06,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T07:18:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:06,889 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:18:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/9dd4316b75034eef837374fcba437dbe is 50, key is test_row_0/A:col10/1731568685899/Put/seqid=0 2024-11-14T07:18:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:18:06,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742297_1473 (size=12151) 2024-11-14T07:18:06,910 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/9dd4316b75034eef837374fcba437dbe 2024-11-14T07:18:06,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/0a1c52181f114e8dacc81d9807f35036 is 50, key is test_row_0/B:col10/1731568685899/Put/seqid=0 2024-11-14T07:18:06,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742298_1474 (size=12151) 2024-11-14T07:18:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:07,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568747103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568747105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568747113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568747202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568747202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,211 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,212 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:18:07,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568747222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568747222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568747232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/0a1c52181f114e8dacc81d9807f35036 2024-11-14T07:18:07,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c8eb3cc2170945df890066097648fc07 is 50, key is test_row_0/C:col10/1731568685899/Put/seqid=0 2024-11-14T07:18:07,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742299_1475 (size=12151) 2024-11-14T07:18:07,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568747432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568747432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568747452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:18:07,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c8eb3cc2170945df890066097648fc07 2024-11-14T07:18:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/9dd4316b75034eef837374fcba437dbe as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/9dd4316b75034eef837374fcba437dbe 2024-11-14T07:18:07,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/9dd4316b75034eef837374fcba437dbe, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T07:18:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/0a1c52181f114e8dacc81d9807f35036 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0a1c52181f114e8dacc81d9807f35036 2024-11-14T07:18:07,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568747752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568747752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,758 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0a1c52181f114e8dacc81d9807f35036, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T07:18:07,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c8eb3cc2170945df890066097648fc07 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c8eb3cc2170945df890066097648fc07 2024-11-14T07:18:07,762 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c8eb3cc2170945df890066097648fc07, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T07:18:07,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6537a8f9e4f66b0d85507b1f78377e39 in 874ms, sequenceid=189, compaction requested=true 2024-11-14T07:18:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T07:18:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T07:18:07,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T07:18:07,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1800 sec 2024-11-14T07:18:07,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.1840 sec 2024-11-14T07:18:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:07,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:18:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:07,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4b191480c8ff414c81efda2332a4aed0 is 50, key is test_row_0/A:col10/1731568687111/Put/seqid=0 2024-11-14T07:18:07,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742300_1476 (size=14541) 2024-11-14T07:18:07,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568747833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:07,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568747942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568748152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:08,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4b191480c8ff414c81efda2332a4aed0 2024-11-14T07:18:08,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8449ea95ed2749349c7d35679a98162e is 50, key is test_row_0/B:col10/1731568687111/Put/seqid=0 2024-11-14T07:18:08,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742301_1477 (size=12151) 2024-11-14T07:18:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568748272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:08,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568748272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568748471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:08,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8449ea95ed2749349c7d35679a98162e 2024-11-14T07:18:08,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/b9496a81a0db49039fe01f3ad439028a is 50, key is test_row_0/C:col10/1731568687111/Put/seqid=0 2024-11-14T07:18:08,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742302_1478 (size=12151) 2024-11-14T07:18:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:18:08,732 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:08,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T07:18:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T07:18:08,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:08,735 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:08,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T07:18:08,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T07:18:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:08,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:08,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568748982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/b9496a81a0db49039fe01f3ad439028a 2024-11-14T07:18:09,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/4b191480c8ff414c81efda2332a4aed0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4b191480c8ff414c81efda2332a4aed0 2024-11-14T07:18:09,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4b191480c8ff414c81efda2332a4aed0, entries=200, sequenceid=209, filesize=14.2 K 2024-11-14T07:18:09,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/8449ea95ed2749349c7d35679a98162e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8449ea95ed2749349c7d35679a98162e 2024-11-14T07:18:09,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8449ea95ed2749349c7d35679a98162e, entries=150, sequenceid=209, filesize=11.9 K 2024-11-14T07:18:09,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/b9496a81a0db49039fe01f3ad439028a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b9496a81a0db49039fe01f3ad439028a 2024-11-14T07:18:09,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b9496a81a0db49039fe01f3ad439028a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-14T07:18:09,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6537a8f9e4f66b0d85507b1f78377e39 in 1256ms, sequenceid=209, compaction requested=true 2024-11-14T07:18:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:09,024 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:09,024 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:09,025 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:09,025 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:09,025 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,025 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c9b088d3c13247a08dfa654b1ab2f39e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4e93d18e16524905948d348eca5664eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/9dd4316b75034eef837374fcba437dbe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4b191480c8ff414c81efda2332a4aed0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=52.5 K 2024-11-14T07:18:09,025 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:09,025 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9b088d3c13247a08dfa654b1ab2f39e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731568683787 2024-11-14T07:18:09,025 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:09,025 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,025 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c5f8a7f288f3465b8d3be83ab1c50233, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2e88447606a4fa49c678a5c85679bea, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0a1c52181f114e8dacc81d9807f35036, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8449ea95ed2749349c7d35679a98162e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=47.9 K 2024-11-14T07:18:09,025 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e93d18e16524905948d348eca5664eb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731568684987 2024-11-14T07:18:09,026 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f8a7f288f3465b8d3be83ab1c50233, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731568683787 2024-11-14T07:18:09,026 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dd4316b75034eef837374fcba437dbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731568685883 2024-11-14T07:18:09,026 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e88447606a4fa49c678a5c85679bea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731568684987 2024-11-14T07:18:09,026 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b191480c8ff414c81efda2332a4aed0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731568687098 2024-11-14T07:18:09,026 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a1c52181f114e8dacc81d9807f35036, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731568685883 2024-11-14T07:18:09,026 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8449ea95ed2749349c7d35679a98162e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731568687103 2024-11-14T07:18:09,034 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:09,034 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/caaca7b956f947f6a2255ab3b7607c42 is 50, key is test_row_0/A:col10/1731568687111/Put/seqid=0 2024-11-14T07:18:09,038 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:09,038 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/a2917945a27b40e38cc19fb1f95b7862 is 50, key is test_row_0/B:col10/1731568687111/Put/seqid=0 2024-11-14T07:18:09,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T07:18:09,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,040 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:18:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T07:18:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/82ee6852812c485bb079eff50f53d028 is 50, key is test_row_0/A:col10/1731568687831/Put/seqid=0 2024-11-14T07:18:09,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742304_1480 (size=12697) 2024-11-14T07:18:09,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742303_1479 (size=12697) 2024-11-14T07:18:09,062 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/caaca7b956f947f6a2255ab3b7607c42 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/caaca7b956f947f6a2255ab3b7607c42 2024-11-14T07:18:09,065 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into caaca7b956f947f6a2255ab3b7607c42(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:09,065 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,065 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=12, startTime=1731568689024; duration=0sec 2024-11-14T07:18:09,065 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:09,065 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:09,065 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:09,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742305_1481 (size=12151) 2024-11-14T07:18:09,067 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:09,068 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:09,068 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,068 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ed1a1a67a75d4fd8b9746ff10d6ecbba, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d49cef310bc840e9a4f50a677103b49e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c8eb3cc2170945df890066097648fc07, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b9496a81a0db49039fe01f3ad439028a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=47.9 K 2024-11-14T07:18:09,068 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1a1a67a75d4fd8b9746ff10d6ecbba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731568683787 2024-11-14T07:18:09,068 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting d49cef310bc840e9a4f50a677103b49e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731568684987 2024-11-14T07:18:09,069 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/82ee6852812c485bb079eff50f53d028 2024-11-14T07:18:09,069 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8eb3cc2170945df890066097648fc07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731568685883 2024-11-14T07:18:09,069 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9496a81a0db49039fe01f3ad439028a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731568687103 2024-11-14T07:18:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/2253612d19e4463788a3870193f2828b is 50, key is test_row_0/B:col10/1731568687831/Put/seqid=0 2024-11-14T07:18:09,086 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:09,087 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/cd2ba7cdb63d46ba9e3eb276cc4ac611 is 50, key is test_row_0/C:col10/1731568687111/Put/seqid=0 2024-11-14T07:18:09,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742306_1482 (size=12151) 2024-11-14T07:18:09,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742307_1483 (size=12697) 2024-11-14T07:18:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:09,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:09,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568749292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568749292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568749295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,299 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,300 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568749298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T07:18:09,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568749402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568749411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,459 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/a2917945a27b40e38cc19fb1f95b7862 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a2917945a27b40e38cc19fb1f95b7862 2024-11-14T07:18:09,463 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into a2917945a27b40e38cc19fb1f95b7862(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:09,463 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,463 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=12, startTime=1731568689024; duration=0sec 2024-11-14T07:18:09,464 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:09,464 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:09,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/2253612d19e4463788a3870193f2828b 2024-11-14T07:18:09,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/5c6ba6919a6b4403ad1458e61bc7f0d0 is 50, key is test_row_0/C:col10/1731568687831/Put/seqid=0 2024-11-14T07:18:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742308_1484 (size=12151) 2024-11-14T07:18:09,501 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/5c6ba6919a6b4403ad1458e61bc7f0d0 2024-11-14T07:18:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/82ee6852812c485bb079eff50f53d028 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/82ee6852812c485bb079eff50f53d028 2024-11-14T07:18:09,509 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/cd2ba7cdb63d46ba9e3eb276cc4ac611 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd2ba7cdb63d46ba9e3eb276cc4ac611 2024-11-14T07:18:09,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/82ee6852812c485bb079eff50f53d028, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T07:18:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/2253612d19e4463788a3870193f2828b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2253612d19e4463788a3870193f2828b 2024-11-14T07:18:09,513 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into cd2ba7cdb63d46ba9e3eb276cc4ac611(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:09,513 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,513 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=12, startTime=1731568689024; duration=0sec 2024-11-14T07:18:09,513 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:09,513 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:09,517 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2253612d19e4463788a3870193f2828b, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T07:18:09,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/5c6ba6919a6b4403ad1458e61bc7f0d0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5c6ba6919a6b4403ad1458e61bc7f0d0 2024-11-14T07:18:09,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5c6ba6919a6b4403ad1458e61bc7f0d0, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T07:18:09,522 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6537a8f9e4f66b0d85507b1f78377e39 in 482ms, sequenceid=221, compaction requested=false 2024-11-14T07:18:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T07:18:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T07:18:09,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T07:18:09,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 788 msec 2024-11-14T07:18:09,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 793 msec 2024-11-14T07:18:09,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T07:18:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:09,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/79c92e35d72547ddb53fd1c2fdae90db is 50, key is test_row_0/A:col10/1731568689269/Put/seqid=0 2024-11-14T07:18:09,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742309_1485 (size=14541) 2024-11-14T07:18:09,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/79c92e35d72547ddb53fd1c2fdae90db 2024-11-14T07:18:09,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/dbfb660edd6a4d8aafa2f76280e76ac0 is 50, key is test_row_0/B:col10/1731568689269/Put/seqid=0 2024-11-14T07:18:09,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742310_1486 (size=12151) 2024-11-14T07:18:09,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/dbfb660edd6a4d8aafa2f76280e76ac0 2024-11-14T07:18:09,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/d0d445b580b14f149a920accf867539b is 50, key is test_row_0/C:col10/1731568689269/Put/seqid=0 2024-11-14T07:18:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568749665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742311_1487 (size=12151) 2024-11-14T07:18:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/d0d445b580b14f149a920accf867539b 2024-11-14T07:18:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568749670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/79c92e35d72547ddb53fd1c2fdae90db as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/79c92e35d72547ddb53fd1c2fdae90db 2024-11-14T07:18:09,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/79c92e35d72547ddb53fd1c2fdae90db, entries=200, sequenceid=245, filesize=14.2 K 2024-11-14T07:18:09,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/dbfb660edd6a4d8aafa2f76280e76ac0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/dbfb660edd6a4d8aafa2f76280e76ac0 2024-11-14T07:18:09,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/dbfb660edd6a4d8aafa2f76280e76ac0, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T07:18:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/d0d445b580b14f149a920accf867539b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d0d445b580b14f149a920accf867539b 2024-11-14T07:18:09,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d0d445b580b14f149a920accf867539b, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T07:18:09,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 6537a8f9e4f66b0d85507b1f78377e39 in 69ms, sequenceid=245, compaction requested=true 2024-11-14T07:18:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:09,697 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:09,697 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:09,698 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:09,698 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:09,698 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,698 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:09,698 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:09,698 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/caaca7b956f947f6a2255ab3b7607c42, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/82ee6852812c485bb079eff50f53d028, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/79c92e35d72547ddb53fd1c2fdae90db] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=38.5 K 2024-11-14T07:18:09,698 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,698 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a2917945a27b40e38cc19fb1f95b7862, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2253612d19e4463788a3870193f2828b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/dbfb660edd6a4d8aafa2f76280e76ac0] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=36.1 K 2024-11-14T07:18:09,699 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting caaca7b956f947f6a2255ab3b7607c42, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731568687103 2024-11-14T07:18:09,699 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting a2917945a27b40e38cc19fb1f95b7862, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731568687103 2024-11-14T07:18:09,700 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 2253612d19e4463788a3870193f2828b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731568687816 2024-11-14T07:18:09,700 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82ee6852812c485bb079eff50f53d028, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731568687816 2024-11-14T07:18:09,702 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79c92e35d72547ddb53fd1c2fdae90db, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731568689269 2024-11-14T07:18:09,702 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfb660edd6a4d8aafa2f76280e76ac0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731568689269 2024-11-14T07:18:09,710 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:09,710 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/3df70e91e92641d8b23aef2ed36d9302 is 50, key is test_row_0/A:col10/1731568689269/Put/seqid=0 2024-11-14T07:18:09,722 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:09,722 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/1fa58396291c453681c9775772d6f1fe is 50, key is test_row_0/B:col10/1731568689269/Put/seqid=0 2024-11-14T07:18:09,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742312_1488 (size=12799) 2024-11-14T07:18:09,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742313_1489 (size=12799) 2024-11-14T07:18:09,738 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/3df70e91e92641d8b23aef2ed36d9302 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3df70e91e92641d8b23aef2ed36d9302 2024-11-14T07:18:09,743 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into 3df70e91e92641d8b23aef2ed36d9302(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:09,744 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,744 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=13, startTime=1731568689697; duration=0sec 2024-11-14T07:18:09,744 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:09,744 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:09,744 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:09,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:09,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:09,745 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:09,745 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd2ba7cdb63d46ba9e3eb276cc4ac611, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5c6ba6919a6b4403ad1458e61bc7f0d0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d0d445b580b14f149a920accf867539b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=36.1 K 2024-11-14T07:18:09,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd2ba7cdb63d46ba9e3eb276cc4ac611, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731568687103 2024-11-14T07:18:09,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c6ba6919a6b4403ad1458e61bc7f0d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731568687816 2024-11-14T07:18:09,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d445b580b14f149a920accf867539b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731568689269 2024-11-14T07:18:09,753 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:09,754 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/87e6c187ca5549c49677646a6779b71e is 50, key is test_row_0/C:col10/1731568689269/Put/seqid=0 2024-11-14T07:18:09,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742314_1490 (size=12799) 2024-11-14T07:18:09,778 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/87e6c187ca5549c49677646a6779b71e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/87e6c187ca5549c49677646a6779b71e 2024-11-14T07:18:09,785 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into 87e6c187ca5549c49677646a6779b71e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:09,785 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:09,785 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=13, startTime=1731568689697; duration=0sec 2024-11-14T07:18:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:09,786 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:09,786 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:09,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:18:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:09,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/f5bc91372eed4340b9f354271314cbbc is 50, key is test_row_0/A:col10/1731568689659/Put/seqid=0 2024-11-14T07:18:09,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742315_1491 (size=14591) 2024-11-14T07:18:09,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/f5bc91372eed4340b9f354271314cbbc 2024-11-14T07:18:09,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/0c719f567a1249e58c1d10f5914e0849 is 50, key is test_row_0/B:col10/1731568689659/Put/seqid=0 2024-11-14T07:18:09,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742316_1492 (size=12201) 2024-11-14T07:18:09,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568749847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568749859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T07:18:09,872 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:09,873 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T07:18:09,874 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:09,875 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:09,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:18:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568749961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568749972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:18:10,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568750002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:10,009 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:10,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:18:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,139 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/1fa58396291c453681c9775772d6f1fe as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/1fa58396291c453681c9775772d6f1fe 2024-11-14T07:18:10,144 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into 1fa58396291c453681c9775772d6f1fe(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:10,144 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:10,144 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=13, startTime=1731568689697; duration=0sec 2024-11-14T07:18:10,144 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:10,144 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568750172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:10,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:18:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:10,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:18:10,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568750192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:10,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/0c719f567a1249e58c1d10f5914e0849 2024-11-14T07:18:10,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/6dcfcee3d41b43e880fd43d957adacd3 is 50, key is test_row_0/C:col10/1731568689659/Put/seqid=0 2024-11-14T07:18:10,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742317_1493 (size=12201) 2024-11-14T07:18:10,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:18:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:18:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568750492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:18:10,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568750501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:10,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/6dcfcee3d41b43e880fd43d957adacd3 2024-11-14T07:18:10,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/f5bc91372eed4340b9f354271314cbbc as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/f5bc91372eed4340b9f354271314cbbc 2024-11-14T07:18:10,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/f5bc91372eed4340b9f354271314cbbc, entries=200, sequenceid=259, filesize=14.2 K 2024-11-14T07:18:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/0c719f567a1249e58c1d10f5914e0849 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0c719f567a1249e58c1d10f5914e0849 2024-11-14T07:18:10,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:18:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:10,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0c719f567a1249e58c1d10f5914e0849, entries=150, sequenceid=259, filesize=11.9 K 2024-11-14T07:18:10,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/6dcfcee3d41b43e880fd43d957adacd3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/6dcfcee3d41b43e880fd43d957adacd3 2024-11-14T07:18:10,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/6dcfcee3d41b43e880fd43d957adacd3, entries=150, sequenceid=259, filesize=11.9 K 2024-11-14T07:18:10,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6537a8f9e4f66b0d85507b1f78377e39 in 862ms, sequenceid=259, compaction requested=false 2024-11-14T07:18:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:10,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:18:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:10,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:18:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:10,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/c762d6fe6090487f9a0f038948ccc2d9 is 50, key is test_row_0/A:col10/1731568689846/Put/seqid=0 2024-11-14T07:18:10,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742318_1494 (size=12301) 2024-11-14T07:18:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:18:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568751048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568751052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568751162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568751162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,201 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/c762d6fe6090487f9a0f038948ccc2d9 2024-11-14T07:18:11,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c70d90c422634931aa9eca803e568202 is 50, key is test_row_0/B:col10/1731568689846/Put/seqid=0 2024-11-14T07:18:11,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742319_1495 (size=12301) 2024-11-14T07:18:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568751312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,322 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568751321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,326 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568751382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568751382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,622 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c70d90c422634931aa9eca803e568202 2024-11-14T07:18:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/480414a7a90f4b7c9421e341cc825a47 is 50, key is test_row_0/C:col10/1731568689846/Put/seqid=0 2024-11-14T07:18:11,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742320_1496 (size=12301) 2024-11-14T07:18:11,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568751702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568751702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:18:12,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568752032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,040 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,045 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/480414a7a90f4b7c9421e341cc825a47 2024-11-14T07:18:12,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/c762d6fe6090487f9a0f038948ccc2d9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c762d6fe6090487f9a0f038948ccc2d9 2024-11-14T07:18:12,050 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c762d6fe6090487f9a0f038948ccc2d9, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T07:18:12,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c70d90c422634931aa9eca803e568202 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c70d90c422634931aa9eca803e568202 2024-11-14T07:18:12,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c70d90c422634931aa9eca803e568202, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T07:18:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/480414a7a90f4b7c9421e341cc825a47 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/480414a7a90f4b7c9421e341cc825a47 2024-11-14T07:18:12,066 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/480414a7a90f4b7c9421e341cc825a47, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T07:18:12,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6537a8f9e4f66b0d85507b1f78377e39 in 1273ms, sequenceid=280, compaction requested=true 2024-11-14T07:18:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T07:18:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T07:18:12,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T07:18:12,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1930 sec 2024-11-14T07:18:12,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1960 sec 2024-11-14T07:18:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:12,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:18:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:12,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/0a88af7dc3dc48e29a206203c385f506 is 50, key is test_row_0/A:col10/1731568692219/Put/seqid=0 2024-11-14T07:18:12,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742321_1497 (size=14741) 2024-11-14T07:18:12,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568752285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568752290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568752402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568752412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/0a88af7dc3dc48e29a206203c385f506 2024-11-14T07:18:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568752621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/f0824931fc654cb6b980cc963ae5304a is 50, key is test_row_0/B:col10/1731568692219/Put/seqid=0 2024-11-14T07:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568752632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742322_1498 (size=12301) 2024-11-14T07:18:12,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568752942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:12,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568752952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:13,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/f0824931fc654cb6b980cc963ae5304a 2024-11-14T07:18:13,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c1f6bd6302fc4a6282bfab20acaf9798 is 50, key is test_row_0/C:col10/1731568692219/Put/seqid=0 2024-11-14T07:18:13,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742323_1499 (size=12301) 2024-11-14T07:18:13,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c1f6bd6302fc4a6282bfab20acaf9798 2024-11-14T07:18:13,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/0a88af7dc3dc48e29a206203c385f506 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0a88af7dc3dc48e29a206203c385f506 2024-11-14T07:18:13,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0a88af7dc3dc48e29a206203c385f506, entries=200, sequenceid=293, filesize=14.4 K 2024-11-14T07:18:13,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/f0824931fc654cb6b980cc963ae5304a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/f0824931fc654cb6b980cc963ae5304a 2024-11-14T07:18:13,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/f0824931fc654cb6b980cc963ae5304a, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T07:18:13,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/c1f6bd6302fc4a6282bfab20acaf9798 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c1f6bd6302fc4a6282bfab20acaf9798 2024-11-14T07:18:13,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568753461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568753461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:13,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c1f6bd6302fc4a6282bfab20acaf9798, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T07:18:13,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6537a8f9e4f66b0d85507b1f78377e39 in 1255ms, sequenceid=293, compaction requested=true 2024-11-14T07:18:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:13,476 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:13,476 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:13,477 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:13,477 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54432 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:13,477 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:13,477 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:13,477 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:13,477 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:13,477 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/1fa58396291c453681c9775772d6f1fe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0c719f567a1249e58c1d10f5914e0849, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c70d90c422634931aa9eca803e568202, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/f0824931fc654cb6b980cc963ae5304a] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=48.4 K 2024-11-14T07:18:13,477 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3df70e91e92641d8b23aef2ed36d9302, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/f5bc91372eed4340b9f354271314cbbc, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c762d6fe6090487f9a0f038948ccc2d9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0a88af7dc3dc48e29a206203c385f506] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=53.2 K 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3df70e91e92641d8b23aef2ed36d9302, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731568689269 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa58396291c453681c9775772d6f1fe, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731568689269 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5bc91372eed4340b9f354271314cbbc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731568689637 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c719f567a1249e58c1d10f5914e0849, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731568689637 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting c762d6fe6090487f9a0f038948ccc2d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568689819 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c70d90c422634931aa9eca803e568202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568689819 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a88af7dc3dc48e29a206203c385f506, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568691032 2024-11-14T07:18:13,478 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f0824931fc654cb6b980cc963ae5304a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568691032 2024-11-14T07:18:13,494 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:13,495 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/b657d962bc954208bc1361849d9b9d30 is 50, key is test_row_0/B:col10/1731568692219/Put/seqid=0 2024-11-14T07:18:13,495 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:13,495 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/69a21b4e0ddd485d81428d66bc428b7e is 50, key is test_row_0/A:col10/1731568692219/Put/seqid=0 2024-11-14T07:18:13,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742324_1500 (size=13085) 2024-11-14T07:18:13,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742325_1501 (size=13085) 2024-11-14T07:18:13,506 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/b657d962bc954208bc1361849d9b9d30 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b657d962bc954208bc1361849d9b9d30 2024-11-14T07:18:13,510 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into b657d962bc954208bc1361849d9b9d30(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:13,510 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:13,510 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=12, startTime=1731568693476; duration=0sec 2024-11-14T07:18:13,510 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:13,510 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:13,510 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:13,511 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:13,511 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:13,511 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:13,511 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/87e6c187ca5549c49677646a6779b71e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/6dcfcee3d41b43e880fd43d957adacd3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/480414a7a90f4b7c9421e341cc825a47, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c1f6bd6302fc4a6282bfab20acaf9798] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=48.4 K 2024-11-14T07:18:13,511 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e6c187ca5549c49677646a6779b71e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731568689269 2024-11-14T07:18:13,512 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dcfcee3d41b43e880fd43d957adacd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731568689637 2024-11-14T07:18:13,512 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 480414a7a90f4b7c9421e341cc825a47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731568689819 2024-11-14T07:18:13,512 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f6bd6302fc4a6282bfab20acaf9798, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568691032 2024-11-14T07:18:13,520 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:13,520 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/71fafe00eff846cfa22d01a57d9b0bad is 50, key is test_row_0/C:col10/1731568692219/Put/seqid=0 2024-11-14T07:18:13,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742326_1502 (size=13085) 2024-11-14T07:18:13,538 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/71fafe00eff846cfa22d01a57d9b0bad as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/71fafe00eff846cfa22d01a57d9b0bad 2024-11-14T07:18:13,544 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into 71fafe00eff846cfa22d01a57d9b0bad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:13,544 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:13,544 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=12, startTime=1731568693476; duration=0sec 2024-11-14T07:18:13,544 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:13,544 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:13,908 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/69a21b4e0ddd485d81428d66bc428b7e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/69a21b4e0ddd485d81428d66bc428b7e 2024-11-14T07:18:13,918 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into 69a21b4e0ddd485d81428d66bc428b7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:13,918 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:13,918 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=12, startTime=1731568693476; duration=0sec 2024-11-14T07:18:13,918 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:13,918 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:18:14,032 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:14,032 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T07:18:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:18:14,034 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:14,034 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:14,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:18:14,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:18:14,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:14,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:14,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/216f974854d341938ebe1f095fd6d635 is 50, key is test_row_0/A:col10/1731568692277/Put/seqid=0 2024-11-14T07:18:14,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742327_1503 (size=12301) 2024-11-14T07:18:14,203 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/216f974854d341938ebe1f095fd6d635 2024-11-14T07:18:14,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c524a75e5cb24f46b27e5682e1386e42 is 50, key is test_row_0/B:col10/1731568692277/Put/seqid=0 2024-11-14T07:18:14,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742328_1504 (size=12301) 2024-11-14T07:18:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:18:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:14,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568754512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568754514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:14,626 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c524a75e5cb24f46b27e5682e1386e42 2024-11-14T07:18:14,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ee4b2f633c74480282d1feda6432af31 is 50, key is test_row_0/C:col10/1731568692277/Put/seqid=0 2024-11-14T07:18:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742329_1505 (size=12301) 2024-11-14T07:18:14,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ee4b2f633c74480282d1feda6432af31 2024-11-14T07:18:14,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/216f974854d341938ebe1f095fd6d635 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/216f974854d341938ebe1f095fd6d635 2024-11-14T07:18:14,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568754633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:14,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568754633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:14,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/216f974854d341938ebe1f095fd6d635, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:18:14,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/c524a75e5cb24f46b27e5682e1386e42 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c524a75e5cb24f46b27e5682e1386e42 2024-11-14T07:18:14,649 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c524a75e5cb24f46b27e5682e1386e42, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:18:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ee4b2f633c74480282d1feda6432af31 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ee4b2f633c74480282d1feda6432af31 2024-11-14T07:18:14,655 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ee4b2f633c74480282d1feda6432af31, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:18:14,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6537a8f9e4f66b0d85507b1f78377e39 in 470ms, sequenceid=315, compaction requested=false 2024-11-14T07:18:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T07:18:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T07:18:14,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T07:18:14,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 623 msec 2024-11-14T07:18:14,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 627 msec 2024-11-14T07:18:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:18:14,662 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:14,662 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-14T07:18:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:18:14,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:14,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:14,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:18:14,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:18:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:14,816 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:18:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:14,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/3c1862e28eb841b4915af1c079ab53fc is 50, key is test_row_0/A:col10/1731568694511/Put/seqid=0 2024-11-14T07:18:14,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742330_1506 (size=12301) 2024-11-14T07:18:14,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/3c1862e28eb841b4915af1c079ab53fc 2024-11-14T07:18:14,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/cb175f1e37724c7fb93a9fa7a00659af is 50, key is test_row_0/B:col10/1731568694511/Put/seqid=0 2024-11-14T07:18:14,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742331_1507 (size=12301) 2024-11-14T07:18:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568754908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:14,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568754916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:18:15,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568755022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568755032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,233 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/cb175f1e37724c7fb93a9fa7a00659af 2024-11-14T07:18:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/b0f74bbabaf543fb8ba95e1d8c5878c1 is 50, key is test_row_0/C:col10/1731568694511/Put/seqid=0 2024-11-14T07:18:15,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568755242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568755242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742332_1508 (size=12301) 2024-11-14T07:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:18:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1731568755342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,347 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56616 deadline: 1731568755352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,358 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in 6537a8f9e4f66b0d85507b1f78377e39 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568755552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568755552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:15,654 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/b0f74bbabaf543fb8ba95e1d8c5878c1 2024-11-14T07:18:15,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/3c1862e28eb841b4915af1c079ab53fc as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3c1862e28eb841b4915af1c079ab53fc 2024-11-14T07:18:15,661 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3c1862e28eb841b4915af1c079ab53fc, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T07:18:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/cb175f1e37724c7fb93a9fa7a00659af as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/cb175f1e37724c7fb93a9fa7a00659af 2024-11-14T07:18:15,664 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/cb175f1e37724c7fb93a9fa7a00659af, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T07:18:15,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/b0f74bbabaf543fb8ba95e1d8c5878c1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b0f74bbabaf543fb8ba95e1d8c5878c1 2024-11-14T07:18:15,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b0f74bbabaf543fb8ba95e1d8c5878c1, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T07:18:15,669 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6537a8f9e4f66b0d85507b1f78377e39 in 853ms, sequenceid=328, compaction requested=true 2024-11-14T07:18:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-14T07:18:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-14T07:18:15,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T07:18:15,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-14T07:18:15,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0100 sec 2024-11-14T07:18:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:18:15,802 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:15,803 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-14T07:18:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:18:15,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:15,807 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:15,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:18:15,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:18:15,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:15,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:15,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:15,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:15,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:15,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:15,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:15,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:15,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/fe8bb2f3077c4a5cbe0a41a340207989 is 50, key is test_row_0/A:col10/1731568694900/Put/seqid=0 2024-11-14T07:18:15,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742333_1509 (size=12301) 2024-11-14T07:18:15,972 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/fe8bb2f3077c4a5cbe0a41a340207989 2024-11-14T07:18:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e750481a771544c4a6722a9bcf160100 is 50, key is test_row_0/B:col10/1731568694900/Put/seqid=0 2024-11-14T07:18:15,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742334_1510 (size=12301) 2024-11-14T07:18:15,982 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e750481a771544c4a6722a9bcf160100 2024-11-14T07:18:15,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/37c04f1467fa41cd8adb84752863688e is 50, key is test_row_0/C:col10/1731568694900/Put/seqid=0 2024-11-14T07:18:15,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742335_1511 (size=12301) 2024-11-14T07:18:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:16,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:16,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568756088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568756090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568756091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:18:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568756202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568756202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568756211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,395 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/37c04f1467fa41cd8adb84752863688e 2024-11-14T07:18:16,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/fe8bb2f3077c4a5cbe0a41a340207989 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/fe8bb2f3077c4a5cbe0a41a340207989 2024-11-14T07:18:16,401 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/fe8bb2f3077c4a5cbe0a41a340207989, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T07:18:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e750481a771544c4a6722a9bcf160100 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e750481a771544c4a6722a9bcf160100 2024-11-14T07:18:16,404 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e750481a771544c4a6722a9bcf160100, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T07:18:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/37c04f1467fa41cd8adb84752863688e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/37c04f1467fa41cd8adb84752863688e 2024-11-14T07:18:16,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/37c04f1467fa41cd8adb84752863688e, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T07:18:16,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6537a8f9e4f66b0d85507b1f78377e39 in 450ms, sequenceid=347, compaction requested=true 2024-11-14T07:18:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-14T07:18:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-14T07:18:16,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T07:18:16,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 604 msec 2024-11-14T07:18:16,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 610 msec 2024-11-14T07:18:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:16,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T07:18:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:18:16,432 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:16,435 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-14T07:18:16,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/e6e763e7a0ea4a99939045339aea89f3 is 50, key is test_row_0/A:col10/1731568696428/Put/seqid=0 2024-11-14T07:18:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:18:16,437 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:16,437 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:16,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:16,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742336_1512 (size=14741) 2024-11-14T07:18:16,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/e6e763e7a0ea4a99939045339aea89f3 2024-11-14T07:18:16,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/54343b54524b49fda402a9d0f9470d79 is 50, key is test_row_0/B:col10/1731568696428/Put/seqid=0 2024-11-14T07:18:16,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742337_1513 (size=12301) 2024-11-14T07:18:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568756479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568756487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568756489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:18:16,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:18:16,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:16,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568756592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568756602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568756611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:18:16,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:16,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:16,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:16,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:18:16,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568756802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568756822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568756831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:16,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/54343b54524b49fda402a9d0f9470d79 2024-11-14T07:18:16,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/4782e38ffead427f834676009177fe53 is 50, key is test_row_0/C:col10/1731568696428/Put/seqid=0 2024-11-14T07:18:16,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742338_1514 (size=12301) 2024-11-14T07:18:16,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:18:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:16,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:17,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:18:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:18:17,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568757112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568757142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568757151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:18:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:17,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/4782e38ffead427f834676009177fe53 2024-11-14T07:18:17,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/e6e763e7a0ea4a99939045339aea89f3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e6e763e7a0ea4a99939045339aea89f3 2024-11-14T07:18:17,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e6e763e7a0ea4a99939045339aea89f3, entries=200, sequenceid=361, filesize=14.4 K 2024-11-14T07:18:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/54343b54524b49fda402a9d0f9470d79 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/54343b54524b49fda402a9d0f9470d79 2024-11-14T07:18:17,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/54343b54524b49fda402a9d0f9470d79, entries=150, sequenceid=361, filesize=12.0 K 2024-11-14T07:18:17,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/4782e38ffead427f834676009177fe53 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/4782e38ffead427f834676009177fe53 2024-11-14T07:18:17,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/4782e38ffead427f834676009177fe53, entries=150, sequenceid=361, filesize=12.0 K 2024-11-14T07:18:17,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6537a8f9e4f66b0d85507b1f78377e39 in 878ms, sequenceid=361, compaction requested=true 2024-11-14T07:18:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:17,308 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:18:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:17,308 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:18:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:17,309 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64729 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:18:17,309 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:17,309 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,309 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/69a21b4e0ddd485d81428d66bc428b7e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/216f974854d341938ebe1f095fd6d635, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3c1862e28eb841b4915af1c079ab53fc, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/fe8bb2f3077c4a5cbe0a41a340207989, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e6e763e7a0ea4a99939045339aea89f3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=63.2 K 2024-11-14T07:18:17,309 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:18:17,309 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:17,309 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,310 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b657d962bc954208bc1361849d9b9d30, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c524a75e5cb24f46b27e5682e1386e42, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/cb175f1e37724c7fb93a9fa7a00659af, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e750481a771544c4a6722a9bcf160100, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/54343b54524b49fda402a9d0f9470d79] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=60.8 K 2024-11-14T07:18:17,313 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting b657d962bc954208bc1361849d9b9d30, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568691032 2024-11-14T07:18:17,313 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a21b4e0ddd485d81428d66bc428b7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568691032 2024-11-14T07:18:17,316 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216f974854d341938ebe1f095fd6d635, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731568692266 2024-11-14T07:18:17,316 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c524a75e5cb24f46b27e5682e1386e42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731568692266 2024-11-14T07:18:17,317 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1862e28eb841b4915af1c079ab53fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731568694503 2024-11-14T07:18:17,317 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe8bb2f3077c4a5cbe0a41a340207989, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731568694900 2024-11-14T07:18:17,317 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cb175f1e37724c7fb93a9fa7a00659af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731568694503 2024-11-14T07:18:17,318 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e763e7a0ea4a99939045339aea89f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731568696077 2024-11-14T07:18:17,318 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e750481a771544c4a6722a9bcf160100, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731568694900 2024-11-14T07:18:17,318 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 54343b54524b49fda402a9d0f9470d79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731568696087 2024-11-14T07:18:17,331 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:17,332 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/7789d1452fe6461c93090798dc991c53 is 50, key is test_row_0/A:col10/1731568696428/Put/seqid=0 2024-11-14T07:18:17,336 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#438 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:17,336 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/ae0be868347a4c4baac7d93d1e3299ac is 50, key is test_row_0/B:col10/1731568696428/Put/seqid=0 2024-11-14T07:18:17,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:18:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T07:18:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:17,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742339_1515 (size=13255) 2024-11-14T07:18:17,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742340_1516 (size=13255) 2024-11-14T07:18:17,366 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/7789d1452fe6461c93090798dc991c53 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7789d1452fe6461c93090798dc991c53 2024-11-14T07:18:17,369 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into 7789d1452fe6461c93090798dc991c53(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:17,369 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:17,369 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=11, startTime=1731568697308; duration=0sec 2024-11-14T07:18:17,369 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:17,370 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:17,370 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:18:17,370 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/ae0be868347a4c4baac7d93d1e3299ac as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/ae0be868347a4c4baac7d93d1e3299ac 2024-11-14T07:18:17,372 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:18:17,372 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:17,372 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:17,372 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/71fafe00eff846cfa22d01a57d9b0bad, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ee4b2f633c74480282d1feda6432af31, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b0f74bbabaf543fb8ba95e1d8c5878c1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/37c04f1467fa41cd8adb84752863688e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/4782e38ffead427f834676009177fe53] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=60.8 K 2024-11-14T07:18:17,372 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71fafe00eff846cfa22d01a57d9b0bad, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731568691032 2024-11-14T07:18:17,373 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee4b2f633c74480282d1feda6432af31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731568692266 2024-11-14T07:18:17,373 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0f74bbabaf543fb8ba95e1d8c5878c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731568694503 2024-11-14T07:18:17,373 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37c04f1467fa41cd8adb84752863688e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731568694900 2024-11-14T07:18:17,374 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4782e38ffead427f834676009177fe53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731568696087 2024-11-14T07:18:17,375 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into ae0be868347a4c4baac7d93d1e3299ac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:17,376 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:17,376 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=11, startTime=1731568697308; duration=0sec 2024-11-14T07:18:17,376 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:17,376 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:17,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/6e3e6d9e27db495c82c4564c9adb2715 is 50, key is test_row_0/A:col10/1731568696477/Put/seqid=0 2024-11-14T07:18:17,424 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#440 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:17,424 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/54cb3b8fd9904edf8d5f687f13e6ecc3 is 50, key is test_row_0/C:col10/1731568696428/Put/seqid=0 2024-11-14T07:18:17,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742341_1517 (size=12301) 2024-11-14T07:18:17,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742342_1518 (size=13255) 2024-11-14T07:18:17,498 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/54cb3b8fd9904edf8d5f687f13e6ecc3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/54cb3b8fd9904edf8d5f687f13e6ecc3 2024-11-14T07:18:17,505 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into 54cb3b8fd9904edf8d5f687f13e6ecc3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:17,506 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:17,506 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=11, startTime=1731568697308; duration=0sec 2024-11-14T07:18:17,506 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:17,506 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:18:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:17,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. as already flushing 2024-11-14T07:18:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568757674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568757675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568757681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568757792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568757792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568757792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:17,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/6e3e6d9e27db495c82c4564c9adb2715 2024-11-14T07:18:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392 is 50, key is test_row_0/B:col10/1731568696477/Put/seqid=0 2024-11-14T07:18:17,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742343_1519 (size=12301) 2024-11-14T07:18:18,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568758002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:18,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568758002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568758012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:18,228 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-14T07:18:18,228 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,228 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,229 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:18,232 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-14T07:18:18,232 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,232 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,232 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:18,237 INFO [Thread-1955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1955. 2024-11-14T07:18:18,237 DEBUG [Thread-1955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,237 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,237 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:18,238 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-14T07:18:18,238 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,238 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,238 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:18,245 INFO [Thread-1953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1953. 2024-11-14T07:18:18,245 DEBUG [Thread-1953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,245 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,245 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:18,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392 2024-11-14T07:18:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/9fcf53c8bdd247a2bd23453bb821eef9 is 50, key is test_row_0/C:col10/1731568696477/Put/seqid=0 2024-11-14T07:18:18,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742344_1520 (size=12301) 2024-11-14T07:18:18,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:18,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56570 deadline: 1731568758311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56590 deadline: 1731568758311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:18,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56582 deadline: 1731568758332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 , the old value is region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:18:18,497 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:18:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:18:18,675 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/9fcf53c8bdd247a2bd23453bb821eef9 2024-11-14T07:18:18,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/6e3e6d9e27db495c82c4564c9adb2715 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6e3e6d9e27db495c82c4564c9adb2715 2024-11-14T07:18:18,681 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6e3e6d9e27db495c82c4564c9adb2715, entries=150, sequenceid=379, filesize=12.0 K 2024-11-14T07:18:18,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392 2024-11-14T07:18:18,683 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392, entries=150, sequenceid=379, filesize=12.0 K 2024-11-14T07:18:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/9fcf53c8bdd247a2bd23453bb821eef9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/9fcf53c8bdd247a2bd23453bb821eef9 2024-11-14T07:18:18,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/9fcf53c8bdd247a2bd23453bb821eef9, entries=150, sequenceid=379, filesize=12.0 K 2024-11-14T07:18:18,687 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6537a8f9e4f66b0d85507b1f78377e39 in 1333ms, sequenceid=379, compaction requested=false 2024-11-14T07:18:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-14T07:18:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-14T07:18:18,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-14T07:18:18,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2510 sec 2024-11-14T07:18:18,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.2540 sec 2024-11-14T07:18:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:18,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T07:18:18,823 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-11-14T07:18:18,823 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,823 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:18,823 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:18,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:18,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:18,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:18,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:18,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:18,824 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-14T07:18:18,824 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,824 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,824 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:18,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/52819b602fa04fc6ab8312c344cc5c6c is 50, key is test_row_0/A:col10/1731568698822/Put/seqid=0 2024-11-14T07:18:18,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742345_1521 (size=12301) 2024-11-14T07:18:18,843 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-14T07:18:18,843 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:18,843 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,843 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:18,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:19,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/52819b602fa04fc6ab8312c344cc5c6c 2024-11-14T07:18:19,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/d38cf03a460d4470ac1b15814991d1e9 is 50, key is test_row_0/B:col10/1731568698822/Put/seqid=0 2024-11-14T07:18:19,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742346_1522 (size=12301) 2024-11-14T07:18:19,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/d38cf03a460d4470ac1b15814991d1e9 2024-11-14T07:18:19,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ac4bc1993dff49159c8892b6ce82ab33 is 50, key is test_row_0/C:col10/1731568698822/Put/seqid=0 2024-11-14T07:18:19,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742347_1523 (size=12301) 2024-11-14T07:18:20,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ac4bc1993dff49159c8892b6ce82ab33 2024-11-14T07:18:20,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/52819b602fa04fc6ab8312c344cc5c6c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/52819b602fa04fc6ab8312c344cc5c6c 2024-11-14T07:18:20,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/52819b602fa04fc6ab8312c344cc5c6c, entries=150, sequenceid=398, filesize=12.0 K 2024-11-14T07:18:20,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/d38cf03a460d4470ac1b15814991d1e9 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/d38cf03a460d4470ac1b15814991d1e9 2024-11-14T07:18:20,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/d38cf03a460d4470ac1b15814991d1e9, entries=150, sequenceid=398, filesize=12.0 K 2024-11-14T07:18:20,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/ac4bc1993dff49159c8892b6ce82ab33 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ac4bc1993dff49159c8892b6ce82ab33 2024-11-14T07:18:20,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ac4bc1993dff49159c8892b6ce82ab33, entries=150, sequenceid=398, filesize=12.0 K 2024-11-14T07:18:20,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for 6537a8f9e4f66b0d85507b1f78377e39 in 1237ms, sequenceid=398, compaction requested=true 2024-11-14T07:18:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:20,060 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6537a8f9e4f66b0d85507b1f78377e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:20,060 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:20,061 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:20,061 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:20,061 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/B is initiating minor compaction (all files) 2024-11-14T07:18:20,061 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/A is initiating minor compaction (all files) 2024-11-14T07:18:20,061 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/B in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:20,061 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/A in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:20,061 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/ae0be868347a4c4baac7d93d1e3299ac, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/d38cf03a460d4470ac1b15814991d1e9] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=37.0 K 2024-11-14T07:18:20,061 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7789d1452fe6461c93090798dc991c53, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6e3e6d9e27db495c82c4564c9adb2715, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/52819b602fa04fc6ab8312c344cc5c6c] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=37.0 K 2024-11-14T07:18:20,062 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0be868347a4c4baac7d93d1e3299ac, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731568696087 2024-11-14T07:18:20,062 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7789d1452fe6461c93090798dc991c53, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731568696087 2024-11-14T07:18:20,062 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e2bd9c1ecd4d4b2fa2fab9950ba1f392, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731568696477 2024-11-14T07:18:20,062 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3e6d9e27db495c82c4564c9adb2715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731568696477 2024-11-14T07:18:20,062 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d38cf03a460d4470ac1b15814991d1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731568697679 2024-11-14T07:18:20,062 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52819b602fa04fc6ab8312c344cc5c6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731568697679 2024-11-14T07:18:20,068 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:20,068 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/fb49e0a58fc848dc97b76dbd5f2e4c22 is 50, key is test_row_0/B:col10/1731568698822/Put/seqid=0 2024-11-14T07:18:20,073 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#A#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:20,073 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/6172139258154f58aab55a8b7e697f07 is 50, key is test_row_0/A:col10/1731568698822/Put/seqid=0 2024-11-14T07:18:20,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742348_1524 (size=13357) 2024-11-14T07:18:20,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742349_1525 (size=13357) 2024-11-14T07:18:20,082 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/6172139258154f58aab55a8b7e697f07 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6172139258154f58aab55a8b7e697f07 2024-11-14T07:18:20,085 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/A of 6537a8f9e4f66b0d85507b1f78377e39 into 6172139258154f58aab55a8b7e697f07(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:20,085 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:20,085 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/A, priority=13, startTime=1731568700060; duration=0sec 2024-11-14T07:18:20,085 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:20,085 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:A 2024-11-14T07:18:20,085 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:20,086 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:20,086 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): 6537a8f9e4f66b0d85507b1f78377e39/C is initiating minor compaction (all files) 2024-11-14T07:18:20,086 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6537a8f9e4f66b0d85507b1f78377e39/C in TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:20,086 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/54cb3b8fd9904edf8d5f687f13e6ecc3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/9fcf53c8bdd247a2bd23453bb821eef9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ac4bc1993dff49159c8892b6ce82ab33] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp, totalSize=37.0 K 2024-11-14T07:18:20,086 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54cb3b8fd9904edf8d5f687f13e6ecc3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731568696087 2024-11-14T07:18:20,086 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fcf53c8bdd247a2bd23453bb821eef9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731568696477 2024-11-14T07:18:20,087 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac4bc1993dff49159c8892b6ce82ab33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731568697679 2024-11-14T07:18:20,092 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6537a8f9e4f66b0d85507b1f78377e39#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:20,092 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/e7ac679b385b40a58863c75573ecbb6d is 50, key is test_row_0/C:col10/1731568698822/Put/seqid=0 2024-11-14T07:18:20,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742350_1526 (size=13357) 2024-11-14T07:18:20,479 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/fb49e0a58fc848dc97b76dbd5f2e4c22 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/fb49e0a58fc848dc97b76dbd5f2e4c22 2024-11-14T07:18:20,482 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/B of 6537a8f9e4f66b0d85507b1f78377e39 into fb49e0a58fc848dc97b76dbd5f2e4c22(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:20,482 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:20,482 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/B, priority=13, startTime=1731568700060; duration=0sec 2024-11-14T07:18:20,482 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:20,482 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:B 2024-11-14T07:18:20,529 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/e7ac679b385b40a58863c75573ecbb6d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/e7ac679b385b40a58863c75573ecbb6d 2024-11-14T07:18:20,532 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6537a8f9e4f66b0d85507b1f78377e39/C of 6537a8f9e4f66b0d85507b1f78377e39 into e7ac679b385b40a58863c75573ecbb6d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:20,532 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6537a8f9e4f66b0d85507b1f78377e39: 2024-11-14T07:18:20,532 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., storeName=6537a8f9e4f66b0d85507b1f78377e39/C, priority=13, startTime=1731568700060; duration=0sec 2024-11-14T07:18:20,532 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:20,532 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6537a8f9e4f66b0d85507b1f78377e39:C 2024-11-14T07:18:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:18:20,602 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:25,393 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-11-14T07:18:25,393 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:25,393 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:25,393 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:25,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:25,453 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-11-14T07:18:25,453 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:25,453 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:25,453 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:25,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1150 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3449 rows 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1161 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3469 rows 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3461 rows 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1150 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3440 rows 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1143 2024-11-14T07:18:25,454 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3428 rows 2024-11-14T07:18:25,454 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:18:25,454 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:18:25,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:25,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:25,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:25,455 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:18:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:18:25,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568705457"}]},"ts":"1731568705457"} 2024-11-14T07:18:25,458 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:18:25,458 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:18:25,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:18:25,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, UNASSIGN}] 2024-11-14T07:18:25,461 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, UNASSIGN 2024-11-14T07:18:25,461 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6537a8f9e4f66b0d85507b1f78377e39, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:25,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, UNASSIGN because future has completed 2024-11-14T07:18:25,463 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:18:25,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:18:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:18:25,614 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:25,614 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 6537a8f9e4f66b0d85507b1f78377e39, disabling compactions & flushes 2024-11-14T07:18:25,615 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. after waiting 0 ms 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:25,615 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 6537a8f9e4f66b0d85507b1f78377e39 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=A 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=B 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6537a8f9e4f66b0d85507b1f78377e39, store=C 2024-11-14T07:18:25,615 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:25,637 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/0e887357e3ce41a4931a4402733f4c40 is 50, key is test_row_0/A:col10/1731568705452/Put/seqid=0 2024-11-14T07:18:25,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742351_1527 (size=12301) 2024-11-14T07:18:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:18:26,041 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/0e887357e3ce41a4931a4402733f4c40 2024-11-14T07:18:26,046 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/b559a9cd53f543a8ad6c347f36f3960c is 50, key is test_row_0/B:col10/1731568705452/Put/seqid=0 2024-11-14T07:18:26,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742352_1528 (size=12301) 2024-11-14T07:18:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:18:26,450 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/b559a9cd53f543a8ad6c347f36f3960c 2024-11-14T07:18:26,455 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/af7f134e8c2a4674b65a261e6c43754f is 50, key is test_row_0/C:col10/1731568705452/Put/seqid=0 2024-11-14T07:18:26,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742353_1529 (size=12301) 2024-11-14T07:18:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:18:26,858 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/af7f134e8c2a4674b65a261e6c43754f 2024-11-14T07:18:26,862 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/A/0e887357e3ce41a4931a4402733f4c40 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0e887357e3ce41a4931a4402733f4c40 2024-11-14T07:18:26,865 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0e887357e3ce41a4931a4402733f4c40, entries=150, sequenceid=407, filesize=12.0 K 2024-11-14T07:18:26,866 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/B/b559a9cd53f543a8ad6c347f36f3960c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b559a9cd53f543a8ad6c347f36f3960c 2024-11-14T07:18:26,868 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b559a9cd53f543a8ad6c347f36f3960c, entries=150, sequenceid=407, filesize=12.0 K 2024-11-14T07:18:26,868 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/.tmp/C/af7f134e8c2a4674b65a261e6c43754f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/af7f134e8c2a4674b65a261e6c43754f 2024-11-14T07:18:26,871 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/af7f134e8c2a4674b65a261e6c43754f, entries=150, sequenceid=407, filesize=12.0 K 2024-11-14T07:18:26,871 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6537a8f9e4f66b0d85507b1f78377e39 in 1256ms, sequenceid=407, compaction requested=false 2024-11-14T07:18:26,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/8de5db060b43465bb33e2b09f7da9538, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/751e0ce35d534a9abbbe03eda45caeb5, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/d8f9a683947b4f7db177149110fc52c8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e0f5322411654ed4836cda696a2ff091, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/95dcf55f924d430fbb3aec0071f6202e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7e96cfbc666d43f584987c53d7b1a0ce, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/11f33930ae054d2fae1d0ab870dc2f18, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/03792df506db48abbe113432779293ee, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/eaacde2458494e1aa43ddc751751ef7e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4f01804b54ad480dae83149d242a5e93, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/2b3d4d1b214646b18b485f97d12dc753, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/bcad04bec9c3479cb2a795a48e8cacbe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c9b088d3c13247a08dfa654b1ab2f39e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4e93d18e16524905948d348eca5664eb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/9dd4316b75034eef837374fcba437dbe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4b191480c8ff414c81efda2332a4aed0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/caaca7b956f947f6a2255ab3b7607c42, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/82ee6852812c485bb079eff50f53d028, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/79c92e35d72547ddb53fd1c2fdae90db, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3df70e91e92641d8b23aef2ed36d9302, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/f5bc91372eed4340b9f354271314cbbc, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c762d6fe6090487f9a0f038948ccc2d9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0a88af7dc3dc48e29a206203c385f506, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/69a21b4e0ddd485d81428d66bc428b7e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/216f974854d341938ebe1f095fd6d635, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3c1862e28eb841b4915af1c079ab53fc, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/fe8bb2f3077c4a5cbe0a41a340207989, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e6e763e7a0ea4a99939045339aea89f3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7789d1452fe6461c93090798dc991c53, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6e3e6d9e27db495c82c4564c9adb2715, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/52819b602fa04fc6ab8312c344cc5c6c] to archive 2024-11-14T07:18:26,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:18:26,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/8de5db060b43465bb33e2b09f7da9538 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/8de5db060b43465bb33e2b09f7da9538 2024-11-14T07:18:26,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/751e0ce35d534a9abbbe03eda45caeb5 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/751e0ce35d534a9abbbe03eda45caeb5 2024-11-14T07:18:26,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/d8f9a683947b4f7db177149110fc52c8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/d8f9a683947b4f7db177149110fc52c8 2024-11-14T07:18:26,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e0f5322411654ed4836cda696a2ff091 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e0f5322411654ed4836cda696a2ff091 2024-11-14T07:18:26,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/95dcf55f924d430fbb3aec0071f6202e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/95dcf55f924d430fbb3aec0071f6202e 2024-11-14T07:18:26,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7e96cfbc666d43f584987c53d7b1a0ce to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7e96cfbc666d43f584987c53d7b1a0ce 2024-11-14T07:18:26,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/11f33930ae054d2fae1d0ab870dc2f18 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/11f33930ae054d2fae1d0ab870dc2f18 2024-11-14T07:18:26,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/03792df506db48abbe113432779293ee to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/03792df506db48abbe113432779293ee 2024-11-14T07:18:26,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/eaacde2458494e1aa43ddc751751ef7e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/eaacde2458494e1aa43ddc751751ef7e 2024-11-14T07:18:26,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4f01804b54ad480dae83149d242a5e93 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4f01804b54ad480dae83149d242a5e93 2024-11-14T07:18:26,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/2b3d4d1b214646b18b485f97d12dc753 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/2b3d4d1b214646b18b485f97d12dc753 2024-11-14T07:18:26,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/bcad04bec9c3479cb2a795a48e8cacbe to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/bcad04bec9c3479cb2a795a48e8cacbe 2024-11-14T07:18:26,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c9b088d3c13247a08dfa654b1ab2f39e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c9b088d3c13247a08dfa654b1ab2f39e 2024-11-14T07:18:26,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4e93d18e16524905948d348eca5664eb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4e93d18e16524905948d348eca5664eb 2024-11-14T07:18:26,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/9dd4316b75034eef837374fcba437dbe to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/9dd4316b75034eef837374fcba437dbe 2024-11-14T07:18:26,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4b191480c8ff414c81efda2332a4aed0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/4b191480c8ff414c81efda2332a4aed0 2024-11-14T07:18:26,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/caaca7b956f947f6a2255ab3b7607c42 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/caaca7b956f947f6a2255ab3b7607c42 2024-11-14T07:18:26,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/82ee6852812c485bb079eff50f53d028 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/82ee6852812c485bb079eff50f53d028 2024-11-14T07:18:26,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/79c92e35d72547ddb53fd1c2fdae90db to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/79c92e35d72547ddb53fd1c2fdae90db 2024-11-14T07:18:26,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3df70e91e92641d8b23aef2ed36d9302 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3df70e91e92641d8b23aef2ed36d9302 2024-11-14T07:18:26,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/f5bc91372eed4340b9f354271314cbbc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/f5bc91372eed4340b9f354271314cbbc 2024-11-14T07:18:26,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c762d6fe6090487f9a0f038948ccc2d9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/c762d6fe6090487f9a0f038948ccc2d9 2024-11-14T07:18:26,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0a88af7dc3dc48e29a206203c385f506 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0a88af7dc3dc48e29a206203c385f506 2024-11-14T07:18:26,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/69a21b4e0ddd485d81428d66bc428b7e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/69a21b4e0ddd485d81428d66bc428b7e 2024-11-14T07:18:26,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/216f974854d341938ebe1f095fd6d635 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/216f974854d341938ebe1f095fd6d635 2024-11-14T07:18:26,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3c1862e28eb841b4915af1c079ab53fc to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/3c1862e28eb841b4915af1c079ab53fc 2024-11-14T07:18:26,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/fe8bb2f3077c4a5cbe0a41a340207989 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/fe8bb2f3077c4a5cbe0a41a340207989 2024-11-14T07:18:26,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e6e763e7a0ea4a99939045339aea89f3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/e6e763e7a0ea4a99939045339aea89f3 2024-11-14T07:18:26,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7789d1452fe6461c93090798dc991c53 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/7789d1452fe6461c93090798dc991c53 2024-11-14T07:18:26,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6e3e6d9e27db495c82c4564c9adb2715 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6e3e6d9e27db495c82c4564c9adb2715 2024-11-14T07:18:26,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/52819b602fa04fc6ab8312c344cc5c6c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/52819b602fa04fc6ab8312c344cc5c6c 2024-11-14T07:18:26,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/110bc9ce30664870b9a2e614b3f86f13, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/77761feab281411f99f4ad1a3eafd56a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c39c91e74ea045bda6feafc8d4c2ec3a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6ce27dad64744b34ba70d3044f22217f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/caeefa8f89eb49b8bc2b707a623c686c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2408815748d1499a9d050d9bef0f34ac, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6a822112811d49c2963f152fa151ba7d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8b8267aacd7a43a28171a6a6a77f58cb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a18a055e1fff483d8cba8ca84913817e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/bf4c06c1f9f2415f9fc998b98735963b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8714205d3a544b9cb808a08323bc5108, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c5f8a7f288f3465b8d3be83ab1c50233, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8d87b662d8cc4f8eaeaf93bf22166d46, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2e88447606a4fa49c678a5c85679bea, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0a1c52181f114e8dacc81d9807f35036, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a2917945a27b40e38cc19fb1f95b7862, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8449ea95ed2749349c7d35679a98162e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2253612d19e4463788a3870193f2828b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/1fa58396291c453681c9775772d6f1fe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/dbfb660edd6a4d8aafa2f76280e76ac0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0c719f567a1249e58c1d10f5914e0849, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c70d90c422634931aa9eca803e568202, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b657d962bc954208bc1361849d9b9d30, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/f0824931fc654cb6b980cc963ae5304a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c524a75e5cb24f46b27e5682e1386e42, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/cb175f1e37724c7fb93a9fa7a00659af, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e750481a771544c4a6722a9bcf160100, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/ae0be868347a4c4baac7d93d1e3299ac, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/54343b54524b49fda402a9d0f9470d79, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/d38cf03a460d4470ac1b15814991d1e9] to archive 2024-11-14T07:18:26,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:18:26,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/110bc9ce30664870b9a2e614b3f86f13 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/110bc9ce30664870b9a2e614b3f86f13 2024-11-14T07:18:26,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/77761feab281411f99f4ad1a3eafd56a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/77761feab281411f99f4ad1a3eafd56a 2024-11-14T07:18:26,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c39c91e74ea045bda6feafc8d4c2ec3a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c39c91e74ea045bda6feafc8d4c2ec3a 2024-11-14T07:18:26,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6ce27dad64744b34ba70d3044f22217f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6ce27dad64744b34ba70d3044f22217f 2024-11-14T07:18:26,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/caeefa8f89eb49b8bc2b707a623c686c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/caeefa8f89eb49b8bc2b707a623c686c 2024-11-14T07:18:26,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2408815748d1499a9d050d9bef0f34ac to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2408815748d1499a9d050d9bef0f34ac 2024-11-14T07:18:26,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6a822112811d49c2963f152fa151ba7d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/6a822112811d49c2963f152fa151ba7d 2024-11-14T07:18:26,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8b8267aacd7a43a28171a6a6a77f58cb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8b8267aacd7a43a28171a6a6a77f58cb 2024-11-14T07:18:26,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a18a055e1fff483d8cba8ca84913817e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a18a055e1fff483d8cba8ca84913817e 2024-11-14T07:18:26,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/bf4c06c1f9f2415f9fc998b98735963b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/bf4c06c1f9f2415f9fc998b98735963b 2024-11-14T07:18:26,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8714205d3a544b9cb808a08323bc5108 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8714205d3a544b9cb808a08323bc5108 2024-11-14T07:18:26,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c5f8a7f288f3465b8d3be83ab1c50233 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c5f8a7f288f3465b8d3be83ab1c50233 2024-11-14T07:18:26,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8d87b662d8cc4f8eaeaf93bf22166d46 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8d87b662d8cc4f8eaeaf93bf22166d46 2024-11-14T07:18:26,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2e88447606a4fa49c678a5c85679bea to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2e88447606a4fa49c678a5c85679bea 2024-11-14T07:18:26,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0a1c52181f114e8dacc81d9807f35036 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0a1c52181f114e8dacc81d9807f35036 2024-11-14T07:18:26,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a2917945a27b40e38cc19fb1f95b7862 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/a2917945a27b40e38cc19fb1f95b7862 2024-11-14T07:18:26,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8449ea95ed2749349c7d35679a98162e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/8449ea95ed2749349c7d35679a98162e 2024-11-14T07:18:26,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2253612d19e4463788a3870193f2828b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/2253612d19e4463788a3870193f2828b 2024-11-14T07:18:26,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/1fa58396291c453681c9775772d6f1fe to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/1fa58396291c453681c9775772d6f1fe 2024-11-14T07:18:26,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/dbfb660edd6a4d8aafa2f76280e76ac0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/dbfb660edd6a4d8aafa2f76280e76ac0 2024-11-14T07:18:26,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0c719f567a1249e58c1d10f5914e0849 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/0c719f567a1249e58c1d10f5914e0849 2024-11-14T07:18:26,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c70d90c422634931aa9eca803e568202 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c70d90c422634931aa9eca803e568202 2024-11-14T07:18:26,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b657d962bc954208bc1361849d9b9d30 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b657d962bc954208bc1361849d9b9d30 2024-11-14T07:18:26,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/f0824931fc654cb6b980cc963ae5304a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/f0824931fc654cb6b980cc963ae5304a 2024-11-14T07:18:26,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c524a75e5cb24f46b27e5682e1386e42 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/c524a75e5cb24f46b27e5682e1386e42 2024-11-14T07:18:26,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/cb175f1e37724c7fb93a9fa7a00659af to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/cb175f1e37724c7fb93a9fa7a00659af 2024-11-14T07:18:26,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e750481a771544c4a6722a9bcf160100 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e750481a771544c4a6722a9bcf160100 2024-11-14T07:18:26,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/ae0be868347a4c4baac7d93d1e3299ac to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/ae0be868347a4c4baac7d93d1e3299ac 2024-11-14T07:18:26,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/54343b54524b49fda402a9d0f9470d79 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/54343b54524b49fda402a9d0f9470d79 2024-11-14T07:18:26,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/e2bd9c1ecd4d4b2fa2fab9950ba1f392 2024-11-14T07:18:26,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/d38cf03a460d4470ac1b15814991d1e9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/d38cf03a460d4470ac1b15814991d1e9 2024-11-14T07:18:26,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/f044b3f0ecc446e285b573b9157d0623, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/10a18fc70dcb4cad9c850a1dd79faa76, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c457368be5284d7fada9a32d4baa110b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/7f00f9ad8b2a454d8c92e98e4831b25b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cb4381c94dea4df1af8c572786bd9977, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd5d0d0f002b4f3f999ae28544b7a044, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a1308205ca784990be61bf5324e6aebe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/22a666669e4b4bd19c9ef1dacf4a6356, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5ac16b0fc6c84e6bbf0a0eb1bc76f383, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/400f6310f6e24cda9e67fd5e3f9e1a73, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a3c8b6dcf7d24aee977f074dbb29c04f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ed1a1a67a75d4fd8b9746ff10d6ecbba, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/44628a7280f64e58955b05d67c8172c2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d49cef310bc840e9a4f50a677103b49e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c8eb3cc2170945df890066097648fc07, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd2ba7cdb63d46ba9e3eb276cc4ac611, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b9496a81a0db49039fe01f3ad439028a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5c6ba6919a6b4403ad1458e61bc7f0d0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/87e6c187ca5549c49677646a6779b71e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d0d445b580b14f149a920accf867539b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/6dcfcee3d41b43e880fd43d957adacd3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/480414a7a90f4b7c9421e341cc825a47, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/71fafe00eff846cfa22d01a57d9b0bad, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c1f6bd6302fc4a6282bfab20acaf9798, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ee4b2f633c74480282d1feda6432af31, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b0f74bbabaf543fb8ba95e1d8c5878c1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/37c04f1467fa41cd8adb84752863688e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/54cb3b8fd9904edf8d5f687f13e6ecc3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/4782e38ffead427f834676009177fe53, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/9fcf53c8bdd247a2bd23453bb821eef9, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ac4bc1993dff49159c8892b6ce82ab33] to archive 2024-11-14T07:18:26,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:18:26,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/f044b3f0ecc446e285b573b9157d0623 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/f044b3f0ecc446e285b573b9157d0623 2024-11-14T07:18:26,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/10a18fc70dcb4cad9c850a1dd79faa76 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/10a18fc70dcb4cad9c850a1dd79faa76 2024-11-14T07:18:26,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c457368be5284d7fada9a32d4baa110b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c457368be5284d7fada9a32d4baa110b 2024-11-14T07:18:26,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/7f00f9ad8b2a454d8c92e98e4831b25b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/7f00f9ad8b2a454d8c92e98e4831b25b 2024-11-14T07:18:26,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cb4381c94dea4df1af8c572786bd9977 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cb4381c94dea4df1af8c572786bd9977 2024-11-14T07:18:26,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd5d0d0f002b4f3f999ae28544b7a044 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd5d0d0f002b4f3f999ae28544b7a044 2024-11-14T07:18:26,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a1308205ca784990be61bf5324e6aebe to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a1308205ca784990be61bf5324e6aebe 2024-11-14T07:18:26,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/22a666669e4b4bd19c9ef1dacf4a6356 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/22a666669e4b4bd19c9ef1dacf4a6356 2024-11-14T07:18:26,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5ac16b0fc6c84e6bbf0a0eb1bc76f383 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5ac16b0fc6c84e6bbf0a0eb1bc76f383 2024-11-14T07:18:26,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/400f6310f6e24cda9e67fd5e3f9e1a73 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/400f6310f6e24cda9e67fd5e3f9e1a73 2024-11-14T07:18:26,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a3c8b6dcf7d24aee977f074dbb29c04f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/a3c8b6dcf7d24aee977f074dbb29c04f 2024-11-14T07:18:26,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ed1a1a67a75d4fd8b9746ff10d6ecbba to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ed1a1a67a75d4fd8b9746ff10d6ecbba 2024-11-14T07:18:26,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/44628a7280f64e58955b05d67c8172c2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/44628a7280f64e58955b05d67c8172c2 2024-11-14T07:18:26,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d49cef310bc840e9a4f50a677103b49e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d49cef310bc840e9a4f50a677103b49e 2024-11-14T07:18:26,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c8eb3cc2170945df890066097648fc07 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c8eb3cc2170945df890066097648fc07 2024-11-14T07:18:26,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd2ba7cdb63d46ba9e3eb276cc4ac611 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/cd2ba7cdb63d46ba9e3eb276cc4ac611 2024-11-14T07:18:26,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b9496a81a0db49039fe01f3ad439028a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b9496a81a0db49039fe01f3ad439028a 2024-11-14T07:18:26,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5c6ba6919a6b4403ad1458e61bc7f0d0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/5c6ba6919a6b4403ad1458e61bc7f0d0 2024-11-14T07:18:26,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/87e6c187ca5549c49677646a6779b71e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/87e6c187ca5549c49677646a6779b71e 2024-11-14T07:18:26,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d0d445b580b14f149a920accf867539b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/d0d445b580b14f149a920accf867539b 2024-11-14T07:18:26,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/6dcfcee3d41b43e880fd43d957adacd3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/6dcfcee3d41b43e880fd43d957adacd3 2024-11-14T07:18:26,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/480414a7a90f4b7c9421e341cc825a47 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/480414a7a90f4b7c9421e341cc825a47 2024-11-14T07:18:26,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/71fafe00eff846cfa22d01a57d9b0bad to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/71fafe00eff846cfa22d01a57d9b0bad 2024-11-14T07:18:26,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c1f6bd6302fc4a6282bfab20acaf9798 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/c1f6bd6302fc4a6282bfab20acaf9798 2024-11-14T07:18:26,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ee4b2f633c74480282d1feda6432af31 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ee4b2f633c74480282d1feda6432af31 2024-11-14T07:18:26,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b0f74bbabaf543fb8ba95e1d8c5878c1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/b0f74bbabaf543fb8ba95e1d8c5878c1 2024-11-14T07:18:26,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/37c04f1467fa41cd8adb84752863688e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/37c04f1467fa41cd8adb84752863688e 2024-11-14T07:18:26,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/54cb3b8fd9904edf8d5f687f13e6ecc3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/54cb3b8fd9904edf8d5f687f13e6ecc3 2024-11-14T07:18:26,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/4782e38ffead427f834676009177fe53 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/4782e38ffead427f834676009177fe53 2024-11-14T07:18:26,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/9fcf53c8bdd247a2bd23453bb821eef9 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/9fcf53c8bdd247a2bd23453bb821eef9 2024-11-14T07:18:26,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ac4bc1993dff49159c8892b6ce82ab33 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/ac4bc1993dff49159c8892b6ce82ab33 2024-11-14T07:18:26,937 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/recovered.edits/410.seqid, newMaxSeqId=410, maxSeqId=1 2024-11-14T07:18:26,937 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. 2024-11-14T07:18:26,937 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 6537a8f9e4f66b0d85507b1f78377e39: Waiting for close lock at 1731568705614Running coprocessor pre-close hooks at 1731568705614Disabling compacts and flushes for region at 1731568705614Disabling writes for close at 1731568705615 (+1 ms)Obtaining lock to block concurrent updates at 1731568705615Preparing flush snapshotting stores in 6537a8f9e4f66b0d85507b1f78377e39 at 1731568705615Finished memstore snapshotting TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731568705615Flushing stores of TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39. at 1731568705635 (+20 ms)Flushing 6537a8f9e4f66b0d85507b1f78377e39/A: creating writer at 1731568705635Flushing 6537a8f9e4f66b0d85507b1f78377e39/A: appending metadata at 1731568705637 (+2 ms)Flushing 6537a8f9e4f66b0d85507b1f78377e39/A: closing flushed file at 1731568705637Flushing 6537a8f9e4f66b0d85507b1f78377e39/B: creating writer at 1731568706044 (+407 ms)Flushing 6537a8f9e4f66b0d85507b1f78377e39/B: appending metadata at 1731568706046 (+2 ms)Flushing 6537a8f9e4f66b0d85507b1f78377e39/B: closing flushed file at 1731568706046Flushing 6537a8f9e4f66b0d85507b1f78377e39/C: creating writer at 1731568706453 (+407 ms)Flushing 6537a8f9e4f66b0d85507b1f78377e39/C: appending metadata at 1731568706455 (+2 ms)Flushing 6537a8f9e4f66b0d85507b1f78377e39/C: closing flushed file at 1731568706455Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ef80d8d: reopening flushed file at 1731568706861 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54036b81: reopening flushed file at 1731568706865 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f4024e7: reopening flushed file at 1731568706868 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6537a8f9e4f66b0d85507b1f78377e39 in 1256ms, sequenceid=407, compaction requested=false at 1731568706871 (+3 ms)Writing region close event to WAL at 1731568706935 (+64 ms)Running coprocessor post-close hooks at 1731568706937 (+2 ms)Closed at 1731568706937 2024-11-14T07:18:26,939 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:26,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6537a8f9e4f66b0d85507b1f78377e39, regionState=CLOSED 2024-11-14T07:18:26,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:18:26,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-14T07:18:26,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 6537a8f9e4f66b0d85507b1f78377e39, server=c7aa6e2790b3,33845,1731568550163 in 1.4780 sec 2024-11-14T07:18:26,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-14T07:18:26,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6537a8f9e4f66b0d85507b1f78377e39, UNASSIGN in 1.4820 sec 2024-11-14T07:18:26,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T07:18:26,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4840 sec 2024-11-14T07:18:26,945 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568706945"}]},"ts":"1731568706945"} 2024-11-14T07:18:26,946 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:18:26,947 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:18:26,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4920 sec 2024-11-14T07:18:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:18:27,602 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:27,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:18:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:27,604 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:18:27,605 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:27,607 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:27,609 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/recovered.edits] 2024-11-14T07:18:27,610 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0e887357e3ce41a4931a4402733f4c40 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/0e887357e3ce41a4931a4402733f4c40 2024-11-14T07:18:27,611 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6172139258154f58aab55a8b7e697f07 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/A/6172139258154f58aab55a8b7e697f07 2024-11-14T07:18:27,612 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b559a9cd53f543a8ad6c347f36f3960c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/b559a9cd53f543a8ad6c347f36f3960c 2024-11-14T07:18:27,613 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/fb49e0a58fc848dc97b76dbd5f2e4c22 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/B/fb49e0a58fc848dc97b76dbd5f2e4c22 2024-11-14T07:18:27,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/af7f134e8c2a4674b65a261e6c43754f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/af7f134e8c2a4674b65a261e6c43754f 2024-11-14T07:18:27,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/e7ac679b385b40a58863c75573ecbb6d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/C/e7ac679b385b40a58863c75573ecbb6d 2024-11-14T07:18:27,617 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/recovered.edits/410.seqid to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39/recovered.edits/410.seqid 2024-11-14T07:18:27,617 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/6537a8f9e4f66b0d85507b1f78377e39 2024-11-14T07:18:27,618 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:18:27,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:27,620 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:18:27,622 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:18:27,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:27,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:18:27,623 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731568707623"}]},"ts":"9223372036854775807"} 2024-11-14T07:18:27,624 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:18:27,624 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6537a8f9e4f66b0d85507b1f78377e39, NAME => 'TestAcidGuarantees,,1731568676039.6537a8f9e4f66b0d85507b1f78377e39.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:18:27,624 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:18:27,624 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731568707624"}]},"ts":"9223372036854775807"} 2024-11-14T07:18:27,626 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:18:27,626 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:27,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 23 msec 2024-11-14T07:18:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:18:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:18:27,715 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:27,720 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 248), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=424 (was 452), ProcessCount=11 (was 11), AvailableMemoryMB=5386 (was 5545) 2024-11-14T07:18:27,727 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=5385 2024-11-14T07:18:27,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:18:27,728 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:18:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:27,730 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:18:27,730 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:27,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-14T07:18:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T07:18:27,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:18:27,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742354_1530 (size=1061) 2024-11-14T07:18:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T07:18:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T07:18:28,136 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22 2024-11-14T07:18:28,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742355_1531 (size=53) 2024-11-14T07:18:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T07:18:28,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:18:28,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cb55791d898838cbae8eb546c5ba678a, disabling compactions & flushes 2024-11-14T07:18:28,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. after waiting 0 ms 2024-11-14T07:18:28,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb55791d898838cbae8eb546c5ba678a: Waiting for close lock at 1731568708541Disabling compacts and flushes for region at 1731568708541Disabling writes for close at 1731568708541Writing region close event to WAL at 1731568708541Closed at 1731568708541 2024-11-14T07:18:28,542 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:18:28,542 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731568708542"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731568708542"}]},"ts":"1731568708542"} 2024-11-14T07:18:28,544 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:18:28,544 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:18:28,544 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568708544"}]},"ts":"1731568708544"} 2024-11-14T07:18:28,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:18:28,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, ASSIGN}] 2024-11-14T07:18:28,546 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, ASSIGN 2024-11-14T07:18:28,547 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=false 2024-11-14T07:18:28,697 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:28,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, ASSIGN because future has completed 2024-11-14T07:18:28,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:18:28,853 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,853 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:18:28,853 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,853 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:18:28,853 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,853 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,854 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,855 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:18:28,855 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb55791d898838cbae8eb546c5ba678a columnFamilyName A 2024-11-14T07:18:28,855 DEBUG [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:28,856 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(327): Store=cb55791d898838cbae8eb546c5ba678a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:18:28,856 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,857 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:18:28,857 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb55791d898838cbae8eb546c5ba678a columnFamilyName B 2024-11-14T07:18:28,857 DEBUG [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:28,857 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(327): Store=cb55791d898838cbae8eb546c5ba678a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:18:28,857 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,858 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:18:28,858 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb55791d898838cbae8eb546c5ba678a columnFamilyName C 2024-11-14T07:18:28,858 DEBUG [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:28,859 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(327): Store=cb55791d898838cbae8eb546c5ba678a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:18:28,859 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,859 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,859 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,860 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,860 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,860 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,861 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:18:28,862 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,863 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:18:28,863 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened cb55791d898838cbae8eb546c5ba678a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71143643, jitterRate=0.06012289226055145}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:18:28,864 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:28,864 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for cb55791d898838cbae8eb546c5ba678a: Running coprocessor pre-open hook at 1731568708853Writing region info on filesystem at 1731568708853Initializing all the Stores at 1731568708854 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568708854Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568708854Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568708854Cleaning up temporary data from old regions at 1731568708860 (+6 ms)Running coprocessor post-open hooks at 1731568708864 (+4 ms)Region opened successfully at 1731568708864 2024-11-14T07:18:28,865 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., pid=144, masterSystemTime=1731568708851 2024-11-14T07:18:28,866 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,866 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:28,867 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:28,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:18:28,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-14T07:18:28,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 in 169 msec 2024-11-14T07:18:28,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-14T07:18:28,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, ASSIGN in 325 msec 2024-11-14T07:18:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T07:18:28,872 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:18:28,872 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568708872"}]},"ts":"1731568708872"} 2024-11-14T07:18:28,873 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:18:28,874 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:18:28,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-14T07:18:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T07:18:29,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:29,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:29,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:29,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:29,893 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:29,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60327653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:29,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:29,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:29,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:29,894 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43444dda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:29,895 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:29,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:29,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:18:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,37073,1731568549456 2024-11-14T07:18:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@e337b67 2024-11-14T07:18:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:18:29,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:18:29,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:18:29,899 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:18:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:29,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742356_1532 (size=1097) 2024-11-14T07:18:30,309 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T07:18:30,310 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T07:18:30,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:18:30,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, REOPEN/MOVE}] 2024-11-14T07:18:30,314 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, REOPEN/MOVE 2024-11-14T07:18:30,315 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, REOPEN/MOVE because future has completed 2024-11-14T07:18:30,316 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:18:30,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:18:30,468 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,468 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:18:30,468 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing cb55791d898838cbae8eb546c5ba678a, disabling compactions & flushes 2024-11-14T07:18:30,468 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,468 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,468 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. after waiting 0 ms 2024-11-14T07:18:30,468 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,471 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T07:18:30,472 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,472 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for cb55791d898838cbae8eb546c5ba678a: Waiting for close lock at 1731568710468Running coprocessor pre-close hooks at 1731568710468Disabling compacts and flushes for region at 1731568710468Disabling writes for close at 1731568710468Writing region close event to WAL at 1731568710468Running coprocessor post-close hooks at 1731568710471 (+3 ms)Closed at 1731568710472 (+1 ms) 2024-11-14T07:18:30,472 WARN [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: cb55791d898838cbae8eb546c5ba678a to self. 2024-11-14T07:18:30,473 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=CLOSED 2024-11-14T07:18:30,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:18:30,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-14T07:18:30,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 in 159 msec 2024-11-14T07:18:30,477 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, REOPEN/MOVE; state=CLOSED, location=c7aa6e2790b3,33845,1731568550163; forceNewPlan=false, retain=true 2024-11-14T07:18:30,628 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=OPENING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, REOPEN/MOVE because future has completed 2024-11-14T07:18:30,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:18:30,815 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,815 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:18:30,816 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,816 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:18:30,816 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,816 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,817 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,818 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:18:30,818 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb55791d898838cbae8eb546c5ba678a columnFamilyName A 2024-11-14T07:18:30,819 DEBUG [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:30,819 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(327): Store=cb55791d898838cbae8eb546c5ba678a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:18:30,819 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,820 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:18:30,820 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb55791d898838cbae8eb546c5ba678a columnFamilyName B 2024-11-14T07:18:30,820 DEBUG [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:30,820 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(327): Store=cb55791d898838cbae8eb546c5ba678a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:18:30,820 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,820 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T07:18:30,820 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb55791d898838cbae8eb546c5ba678a columnFamilyName C 2024-11-14T07:18:30,821 DEBUG [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:30,821 INFO [StoreOpener-cb55791d898838cbae8eb546c5ba678a-1 {}] regionserver.HStore(327): Store=cb55791d898838cbae8eb546c5ba678a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:18:30,821 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,821 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,821 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,822 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,823 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,823 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,823 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:18:30,824 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,825 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened cb55791d898838cbae8eb546c5ba678a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70258253, jitterRate=0.046929553151130676}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:18:30,825 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,825 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for cb55791d898838cbae8eb546c5ba678a: Running coprocessor pre-open hook at 1731568710816Writing region info on filesystem at 1731568710816Initializing all the Stores at 1731568710817 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568710817Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568710817Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731568710817Cleaning up temporary data from old regions at 1731568710823 (+6 ms)Running coprocessor post-open hooks at 1731568710825 (+2 ms)Region opened successfully at 1731568710825 2024-11-14T07:18:30,826 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., pid=149, masterSystemTime=1731568710813 2024-11-14T07:18:30,827 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,827 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:30,828 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=OPEN, openSeqNum=5, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:18:30,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-14T07:18:30,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 in 200 msec 2024-11-14T07:18:30,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T07:18:30,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, REOPEN/MOVE in 517 msec 2024-11-14T07:18:30,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-14T07:18:30,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-14T07:18:30,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-11-14T07:18:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T07:18:30,836 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:30,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,837 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11253cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,838 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f5b9d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,840 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbab468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,840 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a73aab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,842 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755f149c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21503c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9f92a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb0131e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,847 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a44a542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,848 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21924bbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,850 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77575dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,851 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d6fee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,853 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a49f60b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,854 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d69735, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,855 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd112a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3e995e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,858 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78187259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,859 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605b4a3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,37073,-1 for getting cluster id 2024-11-14T07:18:30,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:18:30,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a0774c9-082c-43e3-ac28-5f93eb53c921' 2024-11-14T07:18:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:18:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a0774c9-082c-43e3-ac28-5f93eb53c921" 2024-11-14T07:18:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cff608d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,37073,-1] 2024-11-14T07:18:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:18:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:30,861 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:18:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be22797, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:18:30,862 DEBUG [Thread-2385 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,862 DEBUG [Thread-2379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,862 DEBUG [Thread-2387 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T07:18:30,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:18:30,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:30,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:30,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:30,866 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,866 DEBUG [Thread-2381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,869 DEBUG [Thread-2383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,869 DEBUG [Thread-2394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,871 DEBUG [Thread-2392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,876 DEBUG [Thread-2390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,876 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:18:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,33845,1731568550163, seqNum=-1] 2024-11-14T07:18:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:18:30,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:18:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5] 2024-11-14T07:18:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:30,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:18:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:30,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:30,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:30,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:30,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:30,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:30,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d40520d14bce49f2badaa439fdff8a70_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568710903/Put/seqid=0 2024-11-14T07:18:30,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568770925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568770926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568770928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:30,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568770930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:30,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568770932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:30,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742357_1533 (size=12154) 2024-11-14T07:18:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:18:31,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568771042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568771042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568771042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568771042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568771042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:18:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568771252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568771252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568771252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568771252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568771252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:31,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d40520d14bce49f2badaa439fdff8a70_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d40520d14bce49f2badaa439fdff8a70_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:31,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/5d3b80f9185f4e0ba70b6fce653cc5f2, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:31,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/5d3b80f9185f4e0ba70b6fce653cc5f2 is 175, key is test_row_0/A:col10/1731568710903/Put/seqid=0 2024-11-14T07:18:31,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742358_1534 (size=30955) 2024-11-14T07:18:31,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:31,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:31,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:18:31,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568771562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568771562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568771562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568771562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568771562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:31,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/5d3b80f9185f4e0ba70b6fce653cc5f2 2024-11-14T07:18:31,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/aa60a268eafe482c806744b1d5074b9e is 50, key is test_row_0/B:col10/1731568710903/Put/seqid=0 2024-11-14T07:18:31,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742359_1535 (size=12001) 2024-11-14T07:18:31,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/aa60a268eafe482c806744b1d5074b9e 2024-11-14T07:18:31,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/bac2bbd4ce6445549a98030c3b6980e4 is 50, key is test_row_0/C:col10/1731568710903/Put/seqid=0 2024-11-14T07:18:31,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742360_1536 (size=12001) 2024-11-14T07:18:31,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:31,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:31,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:18:32,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568772072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:32,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568772072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568772072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:32,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568772072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568772072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:32,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:32,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:32,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:32,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/bac2bbd4ce6445549a98030c3b6980e4 2024-11-14T07:18:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/5d3b80f9185f4e0ba70b6fce653cc5f2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/5d3b80f9185f4e0ba70b6fce653cc5f2 2024-11-14T07:18:32,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/5d3b80f9185f4e0ba70b6fce653cc5f2, entries=150, sequenceid=17, filesize=30.2 K 2024-11-14T07:18:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/aa60a268eafe482c806744b1d5074b9e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa60a268eafe482c806744b1d5074b9e 2024-11-14T07:18:32,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa60a268eafe482c806744b1d5074b9e, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T07:18:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/bac2bbd4ce6445549a98030c3b6980e4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/bac2bbd4ce6445549a98030c3b6980e4 2024-11-14T07:18:32,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/bac2bbd4ce6445549a98030c3b6980e4, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T07:18:32,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cb55791d898838cbae8eb546c5ba678a in 1309ms, sequenceid=17, compaction requested=false 2024-11-14T07:18:32,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:32,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T07:18:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:32,240 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:32,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1bd98f49bf143feb34962e72620d5e1_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568710929/Put/seqid=0 2024-11-14T07:18:32,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742361_1537 (size=12154) 2024-11-14T07:18:32,455 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:18:32,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:32,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1bd98f49bf143feb34962e72620d5e1_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1bd98f49bf143feb34962e72620d5e1_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:32,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1fe276f460ed453ca5846a195a097bc3, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:32,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1fe276f460ed453ca5846a195a097bc3 is 175, key is test_row_0/A:col10/1731568710929/Put/seqid=0 2024-11-14T07:18:32,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742362_1538 (size=30955) 2024-11-14T07:18:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:18:33,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1fe276f460ed453ca5846a195a097bc3 2024-11-14T07:18:33,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/8518b4a8f3c74e5c9448fd4f2d160bfe is 50, key is test_row_0/B:col10/1731568710929/Put/seqid=0 2024-11-14T07:18:33,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742363_1539 (size=12001) 2024-11-14T07:18:33,070 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/8518b4a8f3c74e5c9448fd4f2d160bfe 2024-11-14T07:18:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:33,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/17c667c652874e95a2275621df4aee6f is 50, key is test_row_0/C:col10/1731568710929/Put/seqid=0 2024-11-14T07:18:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568773092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568773092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,096 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568773094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568773108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742364_1540 (size=12001) 2024-11-14T07:18:33,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568773109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/17c667c652874e95a2275621df4aee6f 2024-11-14T07:18:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1fe276f460ed453ca5846a195a097bc3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1fe276f460ed453ca5846a195a097bc3 2024-11-14T07:18:33,121 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1fe276f460ed453ca5846a195a097bc3, entries=150, sequenceid=36, filesize=30.2 K 2024-11-14T07:18:33,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/8518b4a8f3c74e5c9448fd4f2d160bfe as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8518b4a8f3c74e5c9448fd4f2d160bfe 2024-11-14T07:18:33,125 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8518b4a8f3c74e5c9448fd4f2d160bfe, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T07:18:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/17c667c652874e95a2275621df4aee6f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/17c667c652874e95a2275621df4aee6f 2024-11-14T07:18:33,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/17c667c652874e95a2275621df4aee6f, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T07:18:33,134 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cb55791d898838cbae8eb546c5ba678a in 893ms, sequenceid=36, compaction requested=false 2024-11-14T07:18:33,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:33,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:33,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T07:18:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T07:18:33,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T07:18:33,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2700 sec 2024-11-14T07:18:33,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2760 sec 2024-11-14T07:18:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:33,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T07:18:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:33,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114867aa92ac16d43b9ba6854b0be6420c0_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568713205/Put/seqid=0 2024-11-14T07:18:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742365_1541 (size=12154) 2024-11-14T07:18:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568773233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568773234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568773235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568773235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568773341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568773342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568773342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568773352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568773551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568773551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568773552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568773562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,614 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:33,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114867aa92ac16d43b9ba6854b0be6420c0_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114867aa92ac16d43b9ba6854b0be6420c0_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:33,618 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/13cc0517ae2d41b98bda10b42279dc58, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:33,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/13cc0517ae2d41b98bda10b42279dc58 is 175, key is test_row_0/A:col10/1731568713205/Put/seqid=0 2024-11-14T07:18:33,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742366_1542 (size=30955) 2024-11-14T07:18:33,621 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/13cc0517ae2d41b98bda10b42279dc58 2024-11-14T07:18:33,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/f1c7ef4daad14d77b70bf69072a7037b is 50, key is test_row_0/B:col10/1731568713205/Put/seqid=0 2024-11-14T07:18:33,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742367_1543 (size=12001) 2024-11-14T07:18:33,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568773862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568773862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568773862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:33,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568773871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:34,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/f1c7ef4daad14d77b70bf69072a7037b 2024-11-14T07:18:34,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/ab05c2ed37304b29bb0eb527b7231621 is 50, key is test_row_0/C:col10/1731568713205/Put/seqid=0 2024-11-14T07:18:34,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742368_1544 (size=12001) 2024-11-14T07:18:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568774372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568774372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568774372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:34,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568774381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:34,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/ab05c2ed37304b29bb0eb527b7231621 2024-11-14T07:18:34,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/13cc0517ae2d41b98bda10b42279dc58 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/13cc0517ae2d41b98bda10b42279dc58 2024-11-14T07:18:34,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/13cc0517ae2d41b98bda10b42279dc58, entries=150, sequenceid=51, filesize=30.2 K 2024-11-14T07:18:34,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/f1c7ef4daad14d77b70bf69072a7037b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1c7ef4daad14d77b70bf69072a7037b 2024-11-14T07:18:34,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1c7ef4daad14d77b70bf69072a7037b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T07:18:34,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/ab05c2ed37304b29bb0eb527b7231621 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ab05c2ed37304b29bb0eb527b7231621 2024-11-14T07:18:34,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ab05c2ed37304b29bb0eb527b7231621, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T07:18:34,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for cb55791d898838cbae8eb546c5ba678a in 1285ms, sequenceid=51, compaction requested=true 2024-11-14T07:18:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:34,491 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:34,491 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:34,495 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:34,495 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:34,495 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/B is initiating minor compaction (all files) 2024-11-14T07:18:34,495 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/A is initiating minor compaction (all files) 2024-11-14T07:18:34,495 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/A in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:34,495 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/B in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:34,495 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/5d3b80f9185f4e0ba70b6fce653cc5f2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1fe276f460ed453ca5846a195a097bc3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/13cc0517ae2d41b98bda10b42279dc58] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=90.7 K 2024-11-14T07:18:34,495 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa60a268eafe482c806744b1d5074b9e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8518b4a8f3c74e5c9448fd4f2d160bfe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1c7ef4daad14d77b70bf69072a7037b] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=35.2 K 2024-11-14T07:18:34,495 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:34,495 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/5d3b80f9185f4e0ba70b6fce653cc5f2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1fe276f460ed453ca5846a195a097bc3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/13cc0517ae2d41b98bda10b42279dc58] 2024-11-14T07:18:34,496 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting aa60a268eafe482c806744b1d5074b9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731568710871 2024-11-14T07:18:34,496 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3b80f9185f4e0ba70b6fce653cc5f2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731568710871 2024-11-14T07:18:34,496 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8518b4a8f3c74e5c9448fd4f2d160bfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731568710921 2024-11-14T07:18:34,496 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fe276f460ed453ca5846a195a097bc3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731568710921 2024-11-14T07:18:34,496 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c7ef4daad14d77b70bf69072a7037b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731568713091 2024-11-14T07:18:34,496 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13cc0517ae2d41b98bda10b42279dc58, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731568713091 2024-11-14T07:18:34,501 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:34,502 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#B#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:34,502 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411143fb29e2766e448ffae4a34a8cc87540c_cb55791d898838cbae8eb546c5ba678a store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:34,502 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/9ccd02bf16d04aa79f75a2b0f5371329 is 50, key is test_row_0/B:col10/1731568713205/Put/seqid=0 2024-11-14T07:18:34,504 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411143fb29e2766e448ffae4a34a8cc87540c_cb55791d898838cbae8eb546c5ba678a, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:34,504 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143fb29e2766e448ffae4a34a8cc87540c_cb55791d898838cbae8eb546c5ba678a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:34,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742369_1545 (size=12104) 2024-11-14T07:18:34,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742370_1546 (size=4469) 2024-11-14T07:18:34,910 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/9ccd02bf16d04aa79f75a2b0f5371329 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/9ccd02bf16d04aa79f75a2b0f5371329 2024-11-14T07:18:34,913 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/B of cb55791d898838cbae8eb546c5ba678a into 9ccd02bf16d04aa79f75a2b0f5371329(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:34,913 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:34,914 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/B, priority=13, startTime=1731568714491; duration=0sec 2024-11-14T07:18:34,914 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:34,914 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:B 2024-11-14T07:18:34,914 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:34,915 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:34,915 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/C is initiating minor compaction (all files) 2024-11-14T07:18:34,915 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/C in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:34,915 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/bac2bbd4ce6445549a98030c3b6980e4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/17c667c652874e95a2275621df4aee6f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ab05c2ed37304b29bb0eb527b7231621] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=35.2 K 2024-11-14T07:18:34,915 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting bac2bbd4ce6445549a98030c3b6980e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731568710871 2024-11-14T07:18:34,916 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c667c652874e95a2275621df4aee6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731568710921 2024-11-14T07:18:34,916 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ab05c2ed37304b29bb0eb527b7231621, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731568713091 2024-11-14T07:18:34,921 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:34,921 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/f41b62a0dac943e4a831f8efa76c6aa0 is 50, key is test_row_0/C:col10/1731568713205/Put/seqid=0 2024-11-14T07:18:34,925 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:34,926 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/639b2e6f81ce4f449f4390d9df67fc51 is 175, key is test_row_0/A:col10/1731568713205/Put/seqid=0 2024-11-14T07:18:34,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742371_1547 (size=12104) 2024-11-14T07:18:34,932 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/f41b62a0dac943e4a831f8efa76c6aa0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f41b62a0dac943e4a831f8efa76c6aa0 2024-11-14T07:18:34,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742372_1548 (size=31058) 2024-11-14T07:18:34,937 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/C of cb55791d898838cbae8eb546c5ba678a into f41b62a0dac943e4a831f8efa76c6aa0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:34,938 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:34,938 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/C, priority=13, startTime=1731568714491; duration=0sec 2024-11-14T07:18:34,938 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:34,938 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:C 2024-11-14T07:18:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:18:35,022 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:35,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T07:18:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T07:18:35,025 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:35,025 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:35,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:35,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T07:18:35,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:35,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:35,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:35,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:35,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:35,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:35,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3dbe25a48024cf980854dfdf329ac2e_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568713233/Put/seqid=0 2024-11-14T07:18:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T07:18:35,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742373_1549 (size=14594) 2024-11-14T07:18:35,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568775144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:35,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:35,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568775252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,337 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/639b2e6f81ce4f449f4390d9df67fc51 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/639b2e6f81ce4f449f4390d9df67fc51 2024-11-14T07:18:35,340 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/A of cb55791d898838cbae8eb546c5ba678a into 639b2e6f81ce4f449f4390d9df67fc51(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:35,340 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:35,340 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/A, priority=13, startTime=1731568714491; duration=0sec 2024-11-14T07:18:35,341 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:35,341 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:A 2024-11-14T07:18:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T07:18:35,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568775382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,384 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568775391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568775391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568775392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,393 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:35,394 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,394 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568775462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:35,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:35,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:35,542 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3dbe25a48024cf980854dfdf329ac2e_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3dbe25a48024cf980854dfdf329ac2e_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:35,543 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/eccbb94d5a2348a7b92442eb40a0eee8, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:35,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/eccbb94d5a2348a7b92442eb40a0eee8 is 175, key is test_row_0/A:col10/1731568713233/Put/seqid=0 2024-11-14T07:18:35,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742374_1550 (size=39549) 2024-11-14T07:18:35,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=72, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/eccbb94d5a2348a7b92442eb40a0eee8 2024-11-14T07:18:35,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/ab7c3aa37a7747459427da8eae86707e is 50, key is test_row_0/B:col10/1731568713233/Put/seqid=0 2024-11-14T07:18:35,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742375_1551 (size=12001) 2024-11-14T07:18:35,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:35,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:35,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T07:18:35,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568775771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:35,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:35,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:35,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:35,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:35,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/ab7c3aa37a7747459427da8eae86707e 2024-11-14T07:18:35,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/f598a2cd3fea42a6956610dbadecab53 is 50, key is test_row_0/C:col10/1731568713233/Put/seqid=0 2024-11-14T07:18:35,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742376_1552 (size=12001) 2024-11-14T07:18:36,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:36,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T07:18:36,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:36,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:36,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568776281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:36,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/f598a2cd3fea42a6956610dbadecab53 2024-11-14T07:18:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/eccbb94d5a2348a7b92442eb40a0eee8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/eccbb94d5a2348a7b92442eb40a0eee8 2024-11-14T07:18:36,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/eccbb94d5a2348a7b92442eb40a0eee8, entries=200, sequenceid=72, filesize=38.6 K 2024-11-14T07:18:36,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/ab7c3aa37a7747459427da8eae86707e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/ab7c3aa37a7747459427da8eae86707e 2024-11-14T07:18:36,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/ab7c3aa37a7747459427da8eae86707e, entries=150, sequenceid=72, filesize=11.7 K 2024-11-14T07:18:36,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/f598a2cd3fea42a6956610dbadecab53 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f598a2cd3fea42a6956610dbadecab53 2024-11-14T07:18:36,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f598a2cd3fea42a6956610dbadecab53, entries=150, sequenceid=72, filesize=11.7 K 2024-11-14T07:18:36,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cb55791d898838cbae8eb546c5ba678a in 1264ms, sequenceid=72, compaction requested=false 2024-11-14T07:18:36,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:36,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T07:18:36,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:36,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T07:18:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d32798bef449447881bce06641b411ad_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568715118/Put/seqid=0 2024-11-14T07:18:36,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742377_1553 (size=12154) 2024-11-14T07:18:36,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:36,411 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d32798bef449447881bce06641b411ad_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d32798bef449447881bce06641b411ad_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/ce831ff4d6534d63b59c748103a720b1, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/ce831ff4d6534d63b59c748103a720b1 is 175, key is test_row_0/A:col10/1731568715118/Put/seqid=0 2024-11-14T07:18:36,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742378_1554 (size=30955) 2024-11-14T07:18:36,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/ce831ff4d6534d63b59c748103a720b1 2024-11-14T07:18:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/4f9b2aac441240ff8ea1fa406dda95d7 is 50, key is test_row_0/B:col10/1731568715118/Put/seqid=0 2024-11-14T07:18:36,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742379_1555 (size=12001) 2024-11-14T07:18:36,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/4f9b2aac441240ff8ea1fa406dda95d7 2024-11-14T07:18:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/21ae655beccc40069c69e3277fe44009 is 50, key is test_row_0/C:col10/1731568715118/Put/seqid=0 2024-11-14T07:18:36,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742380_1556 (size=12001) 2024-11-14T07:18:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T07:18:37,240 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/21ae655beccc40069c69e3277fe44009 2024-11-14T07:18:37,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/ce831ff4d6534d63b59c748103a720b1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/ce831ff4d6534d63b59c748103a720b1 2024-11-14T07:18:37,246 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/ce831ff4d6534d63b59c748103a720b1, entries=150, sequenceid=86, filesize=30.2 K 2024-11-14T07:18:37,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/4f9b2aac441240ff8ea1fa406dda95d7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/4f9b2aac441240ff8ea1fa406dda95d7 2024-11-14T07:18:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,250 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/4f9b2aac441240ff8ea1fa406dda95d7, entries=150, sequenceid=86, filesize=11.7 K 2024-11-14T07:18:37,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/21ae655beccc40069c69e3277fe44009 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/21ae655beccc40069c69e3277fe44009 2024-11-14T07:18:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,254 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/21ae655beccc40069c69e3277fe44009, entries=150, sequenceid=86, filesize=11.7 K 2024-11-14T07:18:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,255 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for cb55791d898838cbae8eb546c5ba678a in 859ms, sequenceid=86, compaction requested=true 2024-11-14T07:18:37,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:37,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:37,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T07:18:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T07:18:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T07:18:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2310 sec 2024-11-14T07:18:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.2360 sec 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:18:37,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:37,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:37,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:37,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:37,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:37,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:37,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d4f1e833c15d4589ba632c3b55f02680_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568717342/Put/seqid=0 2024-11-14T07:18:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742381_1557 (size=14594) 2024-11-14T07:18:37,391 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:37,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d4f1e833c15d4589ba632c3b55f02680_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d4f1e833c15d4589ba632c3b55f02680_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:37,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1a05520483404160abe0bfa20baf1904, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:37,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1a05520483404160abe0bfa20baf1904 is 175, key is test_row_0/A:col10/1731568717342/Put/seqid=0 2024-11-14T07:18:37,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742382_1558 (size=39549) 2024-11-14T07:18:37,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568717405, max timestamp=1731568717419Num uniques 859; 2024-11-14T07:18:37,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568717405, max timestamp=1731568717419Num uniques 859; 2024-11-14T07:18:37,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:18:37,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:18:37,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731568717405, max timestamp=1731568717419Num uniques 859; 2024-11-14T07:18:37,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T07:18:37,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568777439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568777440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568777441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568777442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568777442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568777552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568777552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568777552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568777552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568777552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568777762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568777762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568777762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568777762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568777763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:37,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:37,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:37,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1a05520483404160abe0bfa20baf1904 2024-11-14T07:18:37,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/8178d8fdcde54eecbf244dfcbbee9921 is 50, key is test_row_0/B:col10/1731568717342/Put/seqid=0 2024-11-14T07:18:37,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742383_1559 (size=12001) 2024-11-14T07:18:38,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568778072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568778072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568778072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568778072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568778073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/8178d8fdcde54eecbf244dfcbbee9921 2024-11-14T07:18:38,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/67956568b9864a28bf2bb00686061b76 is 50, key is test_row_0/C:col10/1731568717342/Put/seqid=0 2024-11-14T07:18:38,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742384_1560 (size=12001) 2024-11-14T07:18:38,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568778581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568778581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568778581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568778582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568778592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:38,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/67956568b9864a28bf2bb00686061b76 2024-11-14T07:18:38,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/1a05520483404160abe0bfa20baf1904 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1a05520483404160abe0bfa20baf1904 2024-11-14T07:18:38,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1a05520483404160abe0bfa20baf1904, entries=200, sequenceid=96, filesize=38.6 K 2024-11-14T07:18:38,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/8178d8fdcde54eecbf244dfcbbee9921 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8178d8fdcde54eecbf244dfcbbee9921 2024-11-14T07:18:38,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8178d8fdcde54eecbf244dfcbbee9921, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T07:18:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/67956568b9864a28bf2bb00686061b76 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/67956568b9864a28bf2bb00686061b76 2024-11-14T07:18:38,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/67956568b9864a28bf2bb00686061b76, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T07:18:38,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for cb55791d898838cbae8eb546c5ba678a in 1284ms, sequenceid=96, compaction requested=true 2024-11-14T07:18:38,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:38,641 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:38,642 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:38,642 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:38,643 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/A is initiating minor compaction (all files) 2024-11-14T07:18:38,643 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/A in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:38,643 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/639b2e6f81ce4f449f4390d9df67fc51, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/eccbb94d5a2348a7b92442eb40a0eee8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/ce831ff4d6534d63b59c748103a720b1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1a05520483404160abe0bfa20baf1904] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=137.8 K 2024-11-14T07:18:38,643 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:38,643 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/639b2e6f81ce4f449f4390d9df67fc51, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/eccbb94d5a2348a7b92442eb40a0eee8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/ce831ff4d6534d63b59c748103a720b1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1a05520483404160abe0bfa20baf1904] 2024-11-14T07:18:38,643 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:38,643 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/B is initiating minor compaction (all files) 2024-11-14T07:18:38,643 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/B in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:38,643 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/9ccd02bf16d04aa79f75a2b0f5371329, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/ab7c3aa37a7747459427da8eae86707e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/4f9b2aac441240ff8ea1fa406dda95d7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8178d8fdcde54eecbf244dfcbbee9921] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=47.0 K 2024-11-14T07:18:38,644 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 639b2e6f81ce4f449f4390d9df67fc51, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731568713091 2024-11-14T07:18:38,644 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ccd02bf16d04aa79f75a2b0f5371329, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731568713091 2024-11-14T07:18:38,644 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting eccbb94d5a2348a7b92442eb40a0eee8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731568713211 2024-11-14T07:18:38,644 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ab7c3aa37a7747459427da8eae86707e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731568713211 2024-11-14T07:18:38,644 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce831ff4d6534d63b59c748103a720b1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731568715118 2024-11-14T07:18:38,645 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f9b2aac441240ff8ea1fa406dda95d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731568715118 2024-11-14T07:18:38,645 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a05520483404160abe0bfa20baf1904, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568717342 2024-11-14T07:18:38,646 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 8178d8fdcde54eecbf244dfcbbee9921, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568717342 2024-11-14T07:18:38,664 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:38,664 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/41d49234722145df89e2f645e611b060 is 50, key is test_row_0/B:col10/1731568717342/Put/seqid=0 2024-11-14T07:18:38,665 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:38,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742385_1561 (size=12241) 2024-11-14T07:18:38,693 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148eba47b61d8b410cafbdf46c6788a7b1_cb55791d898838cbae8eb546c5ba678a store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:38,695 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148eba47b61d8b410cafbdf46c6788a7b1_cb55791d898838cbae8eb546c5ba678a, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:38,696 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148eba47b61d8b410cafbdf46c6788a7b1_cb55791d898838cbae8eb546c5ba678a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742386_1562 (size=4469) 2024-11-14T07:18:38,723 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#A#compaction#474 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:38,723 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/832413b43fe14e3da41af8f48d822d3a is 175, key is test_row_0/A:col10/1731568717342/Put/seqid=0 2024-11-14T07:18:38,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742387_1563 (size=31195) 2024-11-14T07:18:38,733 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/832413b43fe14e3da41af8f48d822d3a as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/832413b43fe14e3da41af8f48d822d3a 2024-11-14T07:18:38,738 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/A of cb55791d898838cbae8eb546c5ba678a into 832413b43fe14e3da41af8f48d822d3a(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:38,738 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:38,738 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/A, priority=12, startTime=1731568718641; duration=0sec 2024-11-14T07:18:38,738 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:38,738 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:A 2024-11-14T07:18:38,738 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:18:38,741 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:18:38,741 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/C is initiating minor compaction (all files) 2024-11-14T07:18:38,741 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/C in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:38,741 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f41b62a0dac943e4a831f8efa76c6aa0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f598a2cd3fea42a6956610dbadecab53, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/21ae655beccc40069c69e3277fe44009, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/67956568b9864a28bf2bb00686061b76] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=47.0 K 2024-11-14T07:18:38,741 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f41b62a0dac943e4a831f8efa76c6aa0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731568713091 2024-11-14T07:18:38,742 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f598a2cd3fea42a6956610dbadecab53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731568713211 2024-11-14T07:18:38,742 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ae655beccc40069c69e3277fe44009, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731568715118 2024-11-14T07:18:38,742 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67956568b9864a28bf2bb00686061b76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568717342 2024-11-14T07:18:38,752 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#C#compaction#475 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:38,753 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/927020fd7b984760acbe862d523fab1b is 50, key is test_row_0/C:col10/1731568717342/Put/seqid=0 2024-11-14T07:18:38,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742388_1564 (size=12241) 2024-11-14T07:18:38,774 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/927020fd7b984760acbe862d523fab1b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/927020fd7b984760acbe862d523fab1b 2024-11-14T07:18:38,778 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/C of cb55791d898838cbae8eb546c5ba678a into 927020fd7b984760acbe862d523fab1b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:38,778 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:38,778 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/C, priority=12, startTime=1731568718642; duration=0sec 2024-11-14T07:18:38,778 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:38,778 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:C 2024-11-14T07:18:39,086 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/41d49234722145df89e2f645e611b060 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/41d49234722145df89e2f645e611b060 2024-11-14T07:18:39,090 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/B of cb55791d898838cbae8eb546c5ba678a into 41d49234722145df89e2f645e611b060(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:39,090 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:39,091 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/B, priority=12, startTime=1731568718642; duration=0sec 2024-11-14T07:18:39,091 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:39,091 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:B 2024-11-14T07:18:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T07:18:39,182 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:39,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T07:18:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T07:18:39,184 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:39,185 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:39,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T07:18:39,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T07:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:39,337 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-14T07:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486f0e15fe2f0409383002116d5e3fac9_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568717430/Put/seqid=0 2024-11-14T07:18:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742389_1565 (size=12254) 2024-11-14T07:18:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T07:18:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:39,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:39,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568779602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568779602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,604 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:39,604 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568779603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568779603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568779612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,616 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568779712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568779712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:39,749 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486f0e15fe2f0409383002116d5e3fac9_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486f0e15fe2f0409383002116d5e3fac9_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/dbb3d3483d1c4dacadaf90e15f0467db, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:39,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/dbb3d3483d1c4dacadaf90e15f0467db is 175, key is test_row_0/A:col10/1731568717430/Put/seqid=0 2024-11-14T07:18:39,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742390_1566 (size=31055) 2024-11-14T07:18:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T07:18:39,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568779921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:39,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568779921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:40,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/dbb3d3483d1c4dacadaf90e15f0467db 2024-11-14T07:18:40,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/93ffce38829540539be2d90e1255fa3b is 50, key is test_row_0/B:col10/1731568717430/Put/seqid=0 2024-11-14T07:18:40,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742391_1567 (size=12101) 2024-11-14T07:18:40,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568780232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:40,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568780232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T07:18:40,568 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/93ffce38829540539be2d90e1255fa3b 2024-11-14T07:18:40,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6257472c077f4138b86ac27bd4ce7834 is 50, key is test_row_0/C:col10/1731568717430/Put/seqid=0 2024-11-14T07:18:40,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742392_1568 (size=12101) 2024-11-14T07:18:40,579 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6257472c077f4138b86ac27bd4ce7834 2024-11-14T07:18:40,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/dbb3d3483d1c4dacadaf90e15f0467db as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/dbb3d3483d1c4dacadaf90e15f0467db 2024-11-14T07:18:40,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/dbb3d3483d1c4dacadaf90e15f0467db, entries=150, sequenceid=136, filesize=30.3 K 2024-11-14T07:18:40,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/93ffce38829540539be2d90e1255fa3b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/93ffce38829540539be2d90e1255fa3b 2024-11-14T07:18:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,592 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/93ffce38829540539be2d90e1255fa3b, entries=150, sequenceid=136, filesize=11.8 K 2024-11-14T07:18:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6257472c077f4138b86ac27bd4ce7834 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6257472c077f4138b86ac27bd4ce7834 2024-11-14T07:18:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,595 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6257472c077f4138b86ac27bd4ce7834, entries=150, sequenceid=136, filesize=11.8 K 2024-11-14T07:18:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,595 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for cb55791d898838cbae8eb546c5ba678a in 1258ms, sequenceid=136, compaction requested=false 2024-11-14T07:18:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:40,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:40,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T07:18:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T07:18:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T07:18:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-14T07:18:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4160 sec 2024-11-14T07:18:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:18:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fbec4c11f57749a1af606cc43a470790_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_1/A:col10/1731568720746/Put/seqid=0 2024-11-14T07:18:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742393_1569 (size=12304) 2024-11-14T07:18:40,764 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:40,767 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fbec4c11f57749a1af606cc43a470790_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fbec4c11f57749a1af606cc43a470790_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:40,767 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/09fdb0ce0a6f494baf36214def9b5428, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:40,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/09fdb0ce0a6f494baf36214def9b5428 is 175, key is test_row_1/A:col10/1731568720746/Put/seqid=0 2024-11-14T07:18:40,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742394_1570 (size=31101) 2024-11-14T07:18:40,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568720774, max timestamp=1731568720798Num uniques 859; 2024-11-14T07:18:40,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568720774, max timestamp=1731568720798Num uniques 859; 2024-11-14T07:18:40,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:40,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:40,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568720774, max timestamp=1731568720798Num uniques 859; 2024-11-14T07:18:40,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:40,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568780842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568780845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568780951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568780951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568781162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568781162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/09fdb0ce0a6f494baf36214def9b5428 2024-11-14T07:18:41,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/aa7666d2d1644004a14590cbee7716d4 is 50, key is test_row_1/B:col10/1731568720746/Put/seqid=0 2024-11-14T07:18:41,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742395_1571 (size=9757) 2024-11-14T07:18:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T07:18:41,341 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:41,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T07:18:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T07:18:41,343 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:41,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:41,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T07:18:41,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568781471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568781471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T07:18:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/aa7666d2d1644004a14590cbee7716d4 2024-11-14T07:18:41,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/161c0afed8e648ac9b826a92d1be6121 is 50, key is test_row_1/C:col10/1731568720746/Put/seqid=0 2024-11-14T07:18:41,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742396_1572 (size=9757) 2024-11-14T07:18:41,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568781622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,624 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568781632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,636 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568781642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,645 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T07:18:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T07:18:41,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T07:18:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T07:18:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:41,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T07:18:41,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568781982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568781982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:41,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/161c0afed8e648ac9b826a92d1be6121 2024-11-14T07:18:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/09fdb0ce0a6f494baf36214def9b5428 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/09fdb0ce0a6f494baf36214def9b5428 2024-11-14T07:18:41,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/09fdb0ce0a6f494baf36214def9b5428, entries=150, sequenceid=146, filesize=30.4 K 2024-11-14T07:18:41,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/aa7666d2d1644004a14590cbee7716d4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa7666d2d1644004a14590cbee7716d4 2024-11-14T07:18:42,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa7666d2d1644004a14590cbee7716d4, entries=100, sequenceid=146, filesize=9.5 K 2024-11-14T07:18:42,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/161c0afed8e648ac9b826a92d1be6121 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/161c0afed8e648ac9b826a92d1be6121 2024-11-14T07:18:42,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/161c0afed8e648ac9b826a92d1be6121, entries=100, sequenceid=146, filesize=9.5 K 2024-11-14T07:18:42,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cb55791d898838cbae8eb546c5ba678a in 1256ms, sequenceid=146, compaction requested=true 2024-11-14T07:18:42,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:42,005 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:42,005 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:42,006 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93351 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:42,006 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/A is initiating minor compaction (all files) 2024-11-14T07:18:42,006 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/A in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:42,006 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/832413b43fe14e3da41af8f48d822d3a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/dbb3d3483d1c4dacadaf90e15f0467db, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/09fdb0ce0a6f494baf36214def9b5428] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=91.2 K 2024-11-14T07:18:42,006 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:42,006 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/832413b43fe14e3da41af8f48d822d3a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/dbb3d3483d1c4dacadaf90e15f0467db, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/09fdb0ce0a6f494baf36214def9b5428] 2024-11-14T07:18:42,007 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:42,007 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 832413b43fe14e3da41af8f48d822d3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568717342 2024-11-14T07:18:42,007 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/B is initiating minor compaction (all files) 2024-11-14T07:18:42,007 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/B in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:42,007 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/41d49234722145df89e2f645e611b060, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/93ffce38829540539be2d90e1255fa3b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa7666d2d1644004a14590cbee7716d4] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=33.3 K 2024-11-14T07:18:42,007 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbb3d3483d1c4dacadaf90e15f0467db, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731568717430 2024-11-14T07:18:42,007 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09fdb0ce0a6f494baf36214def9b5428, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568719602 2024-11-14T07:18:42,007 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 41d49234722145df89e2f645e611b060, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568717342 2024-11-14T07:18:42,008 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ffce38829540539be2d90e1255fa3b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731568717430 2024-11-14T07:18:42,008 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting aa7666d2d1644004a14590cbee7716d4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568719602 2024-11-14T07:18:42,013 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:42,015 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:42,015 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/baaf94e08f2e4513980d8fe1cc6b8aae is 50, key is test_row_0/B:col10/1731568717430/Put/seqid=0 2024-11-14T07:18:42,016 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147203f93c7aef46fcb1a53054a31fda7a_cb55791d898838cbae8eb546c5ba678a store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:42,018 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147203f93c7aef46fcb1a53054a31fda7a_cb55791d898838cbae8eb546c5ba678a, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:42,019 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147203f93c7aef46fcb1a53054a31fda7a_cb55791d898838cbae8eb546c5ba678a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:42,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742397_1573 (size=12443) 2024-11-14T07:18:42,026 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/baaf94e08f2e4513980d8fe1cc6b8aae as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/baaf94e08f2e4513980d8fe1cc6b8aae 2024-11-14T07:18:42,031 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/B of cb55791d898838cbae8eb546c5ba678a into baaf94e08f2e4513980d8fe1cc6b8aae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:42,031 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:42,031 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/B, priority=13, startTime=1731568722005; duration=0sec 2024-11-14T07:18:42,031 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:42,031 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:B 2024-11-14T07:18:42,031 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:42,033 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:42,033 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/C is initiating minor compaction (all files) 2024-11-14T07:18:42,033 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/C in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:42,033 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/927020fd7b984760acbe862d523fab1b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6257472c077f4138b86ac27bd4ce7834, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/161c0afed8e648ac9b826a92d1be6121] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=33.3 K 2024-11-14T07:18:42,034 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 927020fd7b984760acbe862d523fab1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731568717342 2024-11-14T07:18:42,034 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6257472c077f4138b86ac27bd4ce7834, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731568717430 2024-11-14T07:18:42,034 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 161c0afed8e648ac9b826a92d1be6121, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568719602 2024-11-14T07:18:42,047 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:42,047 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6f1b8c4e1ffb45eea25cedae37417267 is 50, key is test_row_0/C:col10/1731568717430/Put/seqid=0 2024-11-14T07:18:42,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742398_1574 (size=4469) 2024-11-14T07:18:42,049 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#A#compaction#483 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:42,050 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/9d9a5d9b061541598dac1d12a4707dc6 is 175, key is test_row_0/A:col10/1731568717430/Put/seqid=0 2024-11-14T07:18:42,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742400_1576 (size=31504) 2024-11-14T07:18:42,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742399_1575 (size=12443) 2024-11-14T07:18:42,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T07:18:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:42,106 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T07:18:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:42,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147d40a927e5c4486b8376e429545ca7cf_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568720843/Put/seqid=0 2024-11-14T07:18:42,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742401_1577 (size=12304) 2024-11-14T07:18:42,460 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/9d9a5d9b061541598dac1d12a4707dc6 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/9d9a5d9b061541598dac1d12a4707dc6 2024-11-14T07:18:42,464 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/A of cb55791d898838cbae8eb546c5ba678a into 9d9a5d9b061541598dac1d12a4707dc6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:42,465 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:42,465 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/A, priority=13, startTime=1731568722005; duration=0sec 2024-11-14T07:18:42,465 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6f1b8c4e1ffb45eea25cedae37417267 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6f1b8c4e1ffb45eea25cedae37417267 2024-11-14T07:18:42,465 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:42,465 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:A 2024-11-14T07:18:42,469 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/C of cb55791d898838cbae8eb546c5ba678a into 6f1b8c4e1ffb45eea25cedae37417267(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:42,469 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:42,469 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/C, priority=13, startTime=1731568722005; duration=0sec 2024-11-14T07:18:42,469 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:42,469 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:C 2024-11-14T07:18:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T07:18:42,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,518 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147d40a927e5c4486b8376e429545ca7cf_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147d40a927e5c4486b8376e429545ca7cf_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/92781d7116534c2b8fb8978f27819d52, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/92781d7116534c2b8fb8978f27819d52 is 175, key is test_row_0/A:col10/1731568720843/Put/seqid=0 2024-11-14T07:18:42,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742402_1578 (size=31105) 2024-11-14T07:18:42,522 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/92781d7116534c2b8fb8978f27819d52 2024-11-14T07:18:42,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/d0f97deb757f45cfa3ae434e52a3146d is 50, key is test_row_0/B:col10/1731568720843/Put/seqid=0 2024-11-14T07:18:42,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742403_1579 (size=12151) 2024-11-14T07:18:42,531 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/d0f97deb757f45cfa3ae434e52a3146d 2024-11-14T07:18:42,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9a95c7c4f4074db08521c0b222f67f0c is 50, key is test_row_0/C:col10/1731568720843/Put/seqid=0 2024-11-14T07:18:42,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742404_1580 (size=12151) 2024-11-14T07:18:42,940 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9a95c7c4f4074db08521c0b222f67f0c 2024-11-14T07:18:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/92781d7116534c2b8fb8978f27819d52 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/92781d7116534c2b8fb8978f27819d52 2024-11-14T07:18:42,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/92781d7116534c2b8fb8978f27819d52, entries=150, sequenceid=184, filesize=30.4 K 2024-11-14T07:18:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/d0f97deb757f45cfa3ae434e52a3146d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/d0f97deb757f45cfa3ae434e52a3146d 2024-11-14T07:18:42,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/d0f97deb757f45cfa3ae434e52a3146d, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T07:18:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9a95c7c4f4074db08521c0b222f67f0c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9a95c7c4f4074db08521c0b222f67f0c 2024-11-14T07:18:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9a95c7c4f4074db08521c0b222f67f0c, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for cb55791d898838cbae8eb546c5ba678a in 854ms, sequenceid=184, compaction requested=false 2024-11-14T07:18:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T07:18:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T07:18:42,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-14T07:18:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.6220 sec 2024-11-14T07:18:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:43,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:18:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149cbda0f3e5b0465c8e9a5b8838824061_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568723017/Put/seqid=0 2024-11-14T07:18:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742405_1581 (size=19774) 2024-11-14T07:18:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,041 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149cbda0f3e5b0465c8e9a5b8838824061_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cbda0f3e5b0465c8e9a5b8838824061_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:43,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/630654e609934d62bfc76590ed6a2b0e, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:43,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/630654e609934d62bfc76590ed6a2b0e is 175, key is test_row_0/A:col10/1731568723017/Put/seqid=0 2024-11-14T07:18:43,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742406_1582 (size=57033) 2024-11-14T07:18:43,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568723074, max timestamp=1731568723083Num uniques 859; 2024-11-14T07:18:43,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:43,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568723074, max timestamp=1731568723083Num uniques 859; 2024-11-14T07:18:43,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568723074, max timestamp=1731568723083Num uniques 859; 2024-11-14T07:18:43,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:43,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:43,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568783124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568783129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568783232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568783241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568783442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/630654e609934d62bfc76590ed6a2b0e 2024-11-14T07:18:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568783452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/797b84bb9c3144eca8be71c0f4ca7a50 is 50, key is test_row_0/B:col10/1731568723017/Put/seqid=0 2024-11-14T07:18:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742407_1583 (size=12151) 2024-11-14T07:18:43,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/797b84bb9c3144eca8be71c0f4ca7a50 2024-11-14T07:18:43,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/42b2f57a85654e0ba1aa18641fc3d797 is 50, key is test_row_0/C:col10/1731568723017/Put/seqid=0 2024-11-14T07:18:43,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742408_1584 (size=12151) 2024-11-14T07:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T07:18:43,492 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:43,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-14T07:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:18:43,493 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:43,494 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:43,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:18:43,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:18:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:43,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568783751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568783762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:43,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:18:43,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:43,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:43,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:18:43,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/42b2f57a85654e0ba1aa18641fc3d797 2024-11-14T07:18:43,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/630654e609934d62bfc76590ed6a2b0e as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/630654e609934d62bfc76590ed6a2b0e 2024-11-14T07:18:43,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/630654e609934d62bfc76590ed6a2b0e, entries=300, sequenceid=196, filesize=55.7 K 2024-11-14T07:18:43,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/797b84bb9c3144eca8be71c0f4ca7a50 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/797b84bb9c3144eca8be71c0f4ca7a50 2024-11-14T07:18:43,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/797b84bb9c3144eca8be71c0f4ca7a50, entries=150, sequenceid=196, filesize=11.9 K 2024-11-14T07:18:43,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/42b2f57a85654e0ba1aa18641fc3d797 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/42b2f57a85654e0ba1aa18641fc3d797 2024-11-14T07:18:43,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/42b2f57a85654e0ba1aa18641fc3d797, entries=150, sequenceid=196, filesize=11.9 K 2024-11-14T07:18:43,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cb55791d898838cbae8eb546c5ba678a in 868ms, sequenceid=196, compaction requested=true 2024-11-14T07:18:43,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:43,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:43,900 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:43,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:43,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:43,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:43,900 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:43,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:43,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:43,900 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119642 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:43,900 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/A is initiating minor compaction (all files) 2024-11-14T07:18:43,900 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/A in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,901 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/9d9a5d9b061541598dac1d12a4707dc6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/92781d7116534c2b8fb8978f27819d52, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/630654e609934d62bfc76590ed6a2b0e] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=116.8 K 2024-11-14T07:18:43,901 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,901 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/9d9a5d9b061541598dac1d12a4707dc6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/92781d7116534c2b8fb8978f27819d52, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/630654e609934d62bfc76590ed6a2b0e] 2024-11-14T07:18:43,901 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:43,901 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9a5d9b061541598dac1d12a4707dc6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568717430 2024-11-14T07:18:43,901 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/B is initiating minor compaction (all files) 2024-11-14T07:18:43,901 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/B in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,901 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/baaf94e08f2e4513980d8fe1cc6b8aae, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/d0f97deb757f45cfa3ae434e52a3146d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/797b84bb9c3144eca8be71c0f4ca7a50] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=35.9 K 2024-11-14T07:18:43,901 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92781d7116534c2b8fb8978f27819d52, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568720798 2024-11-14T07:18:43,901 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting baaf94e08f2e4513980d8fe1cc6b8aae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568717430 2024-11-14T07:18:43,902 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 630654e609934d62bfc76590ed6a2b0e, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731568723016 2024-11-14T07:18:43,902 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f97deb757f45cfa3ae434e52a3146d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568720798 2024-11-14T07:18:43,902 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 797b84bb9c3144eca8be71c0f4ca7a50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731568723017 2024-11-14T07:18:43,906 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:43,908 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a7fb395335d54e919e0715ddcc35f58f_cb55791d898838cbae8eb546c5ba678a store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:43,909 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:43,910 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/b07fc80cf96d4914a4696e640be18163 is 50, key is test_row_0/B:col10/1731568723017/Put/seqid=0 2024-11-14T07:18:43,911 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a7fb395335d54e919e0715ddcc35f58f_cb55791d898838cbae8eb546c5ba678a, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:43,911 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a7fb395335d54e919e0715ddcc35f58f_cb55791d898838cbae8eb546c5ba678a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:43,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742409_1585 (size=12595) 2024-11-14T07:18:43,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742410_1586 (size=4469) 2024-11-14T07:18:43,937 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/b07fc80cf96d4914a4696e640be18163 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b07fc80cf96d4914a4696e640be18163 2024-11-14T07:18:43,940 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/B of cb55791d898838cbae8eb546c5ba678a into b07fc80cf96d4914a4696e640be18163(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:43,940 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:43,941 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/B, priority=13, startTime=1731568723900; duration=0sec 2024-11-14T07:18:43,941 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:43,941 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:B 2024-11-14T07:18:43,941 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:43,941 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:43,941 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/C is initiating minor compaction (all files) 2024-11-14T07:18:43,942 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/C in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,942 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6f1b8c4e1ffb45eea25cedae37417267, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9a95c7c4f4074db08521c0b222f67f0c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/42b2f57a85654e0ba1aa18641fc3d797] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=35.9 K 2024-11-14T07:18:43,942 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1b8c4e1ffb45eea25cedae37417267, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731568717430 2024-11-14T07:18:43,943 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a95c7c4f4074db08521c0b222f67f0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731568720798 2024-11-14T07:18:43,943 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b2f57a85654e0ba1aa18641fc3d797, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731568723017 2024-11-14T07:18:43,948 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:43,948 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/08e20a5ebfbf4073a1e492cddd9420f2 is 50, key is test_row_0/C:col10/1731568723017/Put/seqid=0 2024-11-14T07:18:43,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:18:43,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:43,951 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T07:18:43,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:43,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742411_1587 (size=12595) 2024-11-14T07:18:43,962 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/08e20a5ebfbf4073a1e492cddd9420f2 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/08e20a5ebfbf4073a1e492cddd9420f2 2024-11-14T07:18:43,967 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/C of cb55791d898838cbae8eb546c5ba678a into 08e20a5ebfbf4073a1e492cddd9420f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:43,967 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:43,967 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/C, priority=13, startTime=1731568723900; duration=0sec 2024-11-14T07:18:43,967 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:43,967 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:C 2024-11-14T07:18:43,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b593d36a9c94232a6f7b6bad2b95ff0_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568723123/Put/seqid=0 2024-11-14T07:18:43,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742412_1588 (size=12304) 2024-11-14T07:18:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:18:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:44,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:44,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568784280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:44,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568784285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:44,334 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#A#compaction#491 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:44,335 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/6f58a91c060f4e7587efbbd4955754fb is 175, key is test_row_0/A:col10/1731568723017/Put/seqid=0 2024-11-14T07:18:44,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742413_1589 (size=31549) 2024-11-14T07:18:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:44,381 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b593d36a9c94232a6f7b6bad2b95ff0_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b593d36a9c94232a6f7b6bad2b95ff0_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/6cf6a775434d4ba49c959197f66732d8, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:44,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/6cf6a775434d4ba49c959197f66732d8 is 175, key is test_row_0/A:col10/1731568723123/Put/seqid=0 2024-11-14T07:18:44,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742414_1590 (size=31105) 2024-11-14T07:18:44,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568784391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:44,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568784402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:44,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568784602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568784611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:18:44,742 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/6f58a91c060f4e7587efbbd4955754fb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6f58a91c060f4e7587efbbd4955754fb 2024-11-14T07:18:44,745 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/A of cb55791d898838cbae8eb546c5ba678a into 6f58a91c060f4e7587efbbd4955754fb(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:44,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:44,745 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/A, priority=13, startTime=1731568723900; duration=0sec 2024-11-14T07:18:44,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:44,745 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:A 2024-11-14T07:18:44,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/6cf6a775434d4ba49c959197f66732d8 2024-11-14T07:18:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/e54d30f49e2f4099bc8109691f636ae1 is 50, key is test_row_0/B:col10/1731568723123/Put/seqid=0 2024-11-14T07:18:44,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742415_1591 (size=12151) 2024-11-14T07:18:44,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568784912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:44,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568784922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,196 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/e54d30f49e2f4099bc8109691f636ae1 2024-11-14T07:18:45,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/3e06261989ed4028aac0ac4b74598b42 is 50, key is test_row_0/C:col10/1731568723123/Put/seqid=0 2024-11-14T07:18:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742416_1592 (size=12151) 2024-11-14T07:18:45,215 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/3e06261989ed4028aac0ac4b74598b42 2024-11-14T07:18:45,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/6cf6a775434d4ba49c959197f66732d8 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6cf6a775434d4ba49c959197f66732d8 2024-11-14T07:18:45,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6cf6a775434d4ba49c959197f66732d8, entries=150, sequenceid=234, filesize=30.4 K 2024-11-14T07:18:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/e54d30f49e2f4099bc8109691f636ae1 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/e54d30f49e2f4099bc8109691f636ae1 2024-11-14T07:18:45,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/e54d30f49e2f4099bc8109691f636ae1, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:18:45,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/3e06261989ed4028aac0ac4b74598b42 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/3e06261989ed4028aac0ac4b74598b42 2024-11-14T07:18:45,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/3e06261989ed4028aac0ac4b74598b42, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:18:45,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cb55791d898838cbae8eb546c5ba678a in 1286ms, sequenceid=234, compaction requested=false 2024-11-14T07:18:45,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:45,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-14T07:18:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-14T07:18:45,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T07:18:45,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7460 sec 2024-11-14T07:18:45,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.7510 sec 2024-11-14T07:18:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:45,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T07:18:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:45,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143fe5101c353f4e449d31fbfbcb85b019_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568725424/Put/seqid=0 2024-11-14T07:18:45,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742417_1593 (size=14794) 2024-11-14T07:18:45,437 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:45,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143fe5101c353f4e449d31fbfbcb85b019_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143fe5101c353f4e449d31fbfbcb85b019_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:45,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8a968c3816ad4bf6b696d1faca0da403, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:45,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8a968c3816ad4bf6b696d1faca0da403 is 175, key is test_row_0/A:col10/1731568725424/Put/seqid=0 2024-11-14T07:18:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742418_1594 (size=39749) 2024-11-14T07:18:45,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8a968c3816ad4bf6b696d1faca0da403 2024-11-14T07:18:45,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/19a5998101b446adac97a48289644a56 is 50, key is test_row_0/B:col10/1731568725424/Put/seqid=0 2024-11-14T07:18:45,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742419_1595 (size=12151) 2024-11-14T07:18:45,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/19a5998101b446adac97a48289644a56 2024-11-14T07:18:45,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/c6a9da62c0b44d2a8b1efef57c8bfbed is 50, key is test_row_0/C:col10/1731568725424/Put/seqid=0 2024-11-14T07:18:45,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742420_1596 (size=12151) 2024-11-14T07:18:45,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568785473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568725460, max timestamp=1731568725477Num uniques 859; 2024-11-14T07:18:45,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568725460, max timestamp=1731568725477Num uniques 859; 2024-11-14T07:18:45,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:45,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:45,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568725460, max timestamp=1731568725477Num uniques 859; 2024-11-14T07:18:45,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568785553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568785582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38624 deadline: 1731568785632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,634 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:18:45,642 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:45,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-14T07:18:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:18:45,644 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:45,644 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:45,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:45,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1731568785662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568785662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38616 deadline: 1731568785662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,667 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_0' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,667 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_1' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:18:45,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568785791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T07:18:45,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:45,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:45,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/c6a9da62c0b44d2a8b1efef57c8bfbed 2024-11-14T07:18:45,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8a968c3816ad4bf6b696d1faca0da403 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8a968c3816ad4bf6b696d1faca0da403 2024-11-14T07:18:45,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:45,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8a968c3816ad4bf6b696d1faca0da403, entries=200, sequenceid=247, filesize=38.8 K 2024-11-14T07:18:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568785872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:45,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/19a5998101b446adac97a48289644a56 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/19a5998101b446adac97a48289644a56 2024-11-14T07:18:45,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/19a5998101b446adac97a48289644a56, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T07:18:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/c6a9da62c0b44d2a8b1efef57c8bfbed as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/c6a9da62c0b44d2a8b1efef57c8bfbed 2024-11-14T07:18:45,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/c6a9da62c0b44d2a8b1efef57c8bfbed, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T07:18:45,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cb55791d898838cbae8eb546c5ba678a in 457ms, sequenceid=247, compaction requested=true 2024-11-14T07:18:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:45,882 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:18:45,883 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:45,888 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:45,888 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:45,888 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/B is initiating minor compaction (all files) 2024-11-14T07:18:45,888 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/A is initiating minor compaction (all files) 2024-11-14T07:18:45,889 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/B in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,889 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/A in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,889 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b07fc80cf96d4914a4696e640be18163, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/e54d30f49e2f4099bc8109691f636ae1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/19a5998101b446adac97a48289644a56] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=36.0 K 2024-11-14T07:18:45,889 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6f58a91c060f4e7587efbbd4955754fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6cf6a775434d4ba49c959197f66732d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8a968c3816ad4bf6b696d1faca0da403] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=100.0 K 2024-11-14T07:18:45,889 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,889 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6f58a91c060f4e7587efbbd4955754fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6cf6a775434d4ba49c959197f66732d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8a968c3816ad4bf6b696d1faca0da403] 2024-11-14T07:18:45,889 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting b07fc80cf96d4914a4696e640be18163, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731568723017 2024-11-14T07:18:45,889 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f58a91c060f4e7587efbbd4955754fb, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731568723017 2024-11-14T07:18:45,889 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting e54d30f49e2f4099bc8109691f636ae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731568723122 2024-11-14T07:18:45,889 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a5998101b446adac97a48289644a56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568724273 2024-11-14T07:18:45,889 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf6a775434d4ba49c959197f66732d8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731568723122 2024-11-14T07:18:45,890 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a968c3816ad4bf6b696d1faca0da403, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568724273 2024-11-14T07:18:45,898 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:45,898 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/7f235ee292c449ee8b3b364e87f69db7 is 50, key is test_row_0/B:col10/1731568725424/Put/seqid=0 2024-11-14T07:18:45,905 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:45,921 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f7639f1d33a843b8a70cd3bfd9cf54e9_cb55791d898838cbae8eb546c5ba678a store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:45,923 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f7639f1d33a843b8a70cd3bfd9cf54e9_cb55791d898838cbae8eb546c5ba678a, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:45,923 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f7639f1d33a843b8a70cd3bfd9cf54e9_cb55791d898838cbae8eb546c5ba678a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:45,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742421_1597 (size=12697) 2024-11-14T07:18:45,942 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/7f235ee292c449ee8b3b364e87f69db7 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/7f235ee292c449ee8b3b364e87f69db7 2024-11-14T07:18:45,948 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/B of cb55791d898838cbae8eb546c5ba678a into 7f235ee292c449ee8b3b364e87f69db7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:45,948 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:45,948 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/B, priority=13, startTime=1731568725882; duration=0sec 2024-11-14T07:18:45,948 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:45,948 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:B 2024-11-14T07:18:45,948 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:45,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T07:18:45,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,949 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-14T07:18:45,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:45,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:45,952 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:45,952 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/C is initiating minor compaction (all files) 2024-11-14T07:18:45,952 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/C in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:45,952 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/08e20a5ebfbf4073a1e492cddd9420f2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/3e06261989ed4028aac0ac4b74598b42, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/c6a9da62c0b44d2a8b1efef57c8bfbed] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=36.0 K 2024-11-14T07:18:45,952 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e20a5ebfbf4073a1e492cddd9420f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731568723017 2024-11-14T07:18:45,953 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e06261989ed4028aac0ac4b74598b42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731568723122 2024-11-14T07:18:45,953 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a9da62c0b44d2a8b1efef57c8bfbed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568724273 2024-11-14T07:18:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:18:45,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742422_1598 (size=4469) 2024-11-14T07:18:45,974 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:45,975 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/01fffd694d984a85ae6a9a7eecbd6bac is 50, key is test_row_0/C:col10/1731568725424/Put/seqid=0 2024-11-14T07:18:45,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742423_1599 (size=12697) 2024-11-14T07:18:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148e10b96c24ff4eb89f1ee650e8b3f2a1_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568725530/Put/seqid=0 2024-11-14T07:18:45,983 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/01fffd694d984a85ae6a9a7eecbd6bac as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/01fffd694d984a85ae6a9a7eecbd6bac 2024-11-14T07:18:45,988 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/C of cb55791d898838cbae8eb546c5ba678a into 01fffd694d984a85ae6a9a7eecbd6bac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:45,988 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:45,988 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/C, priority=13, startTime=1731568725882; duration=0sec 2024-11-14T07:18:45,988 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:45,988 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:C 2024-11-14T07:18:45,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742424_1600 (size=12454) 2024-11-14T07:18:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:46,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:46,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568786140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568786181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:46,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568786252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:18:46,366 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#A#compaction#501 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:46,367 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/e64624975a634223babba205e92cef1d is 175, key is test_row_0/A:col10/1731568725424/Put/seqid=0 2024-11-14T07:18:46,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742425_1601 (size=31651) 2024-11-14T07:18:46,373 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/e64624975a634223babba205e92cef1d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/e64624975a634223babba205e92cef1d 2024-11-14T07:18:46,376 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/A of cb55791d898838cbae8eb546c5ba678a into e64624975a634223babba205e92cef1d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:46,376 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:46,376 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/A, priority=13, startTime=1731568725882; duration=0sec 2024-11-14T07:18:46,376 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:46,376 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:A 2024-11-14T07:18:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:46,397 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148e10b96c24ff4eb89f1ee650e8b3f2a1_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148e10b96c24ff4eb89f1ee650e8b3f2a1_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:46,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/2a794ca28fb944fda2e558be4af9be56, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:46,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/2a794ca28fb944fda2e558be4af9be56 is 175, key is test_row_0/A:col10/1731568725530/Put/seqid=0 2024-11-14T07:18:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742426_1602 (size=31255) 2024-11-14T07:18:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568786462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:46,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568786691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568786772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:18:46,801 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/2a794ca28fb944fda2e558be4af9be56 2024-11-14T07:18:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/05f20d1f789649418a36e688200b81b0 is 50, key is test_row_0/B:col10/1731568725530/Put/seqid=0 2024-11-14T07:18:46,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742427_1603 (size=12301) 2024-11-14T07:18:46,812 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/05f20d1f789649418a36e688200b81b0 2024-11-14T07:18:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9fcf8507fdfd4434bba30e09804015bf is 50, key is test_row_0/C:col10/1731568725530/Put/seqid=0 2024-11-14T07:18:46,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742428_1604 (size=12301) 2024-11-14T07:18:46,822 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9fcf8507fdfd4434bba30e09804015bf 2024-11-14T07:18:46,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/2a794ca28fb944fda2e558be4af9be56 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/2a794ca28fb944fda2e558be4af9be56 2024-11-14T07:18:46,829 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/2a794ca28fb944fda2e558be4af9be56, entries=150, sequenceid=284, filesize=30.5 K 2024-11-14T07:18:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/05f20d1f789649418a36e688200b81b0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/05f20d1f789649418a36e688200b81b0 2024-11-14T07:18:46,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/05f20d1f789649418a36e688200b81b0, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T07:18:46,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9fcf8507fdfd4434bba30e09804015bf as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9fcf8507fdfd4434bba30e09804015bf 2024-11-14T07:18:46,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9fcf8507fdfd4434bba30e09804015bf, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T07:18:46,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cb55791d898838cbae8eb546c5ba678a in 888ms, sequenceid=284, compaction requested=false 2024-11-14T07:18:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-14T07:18:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-14T07:18:46,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T07:18:46,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1930 sec 2024-11-14T07:18:46,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.1980 sec 2024-11-14T07:18:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:47,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T07:18:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:47,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114894a995481cb4203ac2a1b3c96cbbc27_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568726139/Put/seqid=0 2024-11-14T07:18:47,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742429_1605 (size=14994) 2024-11-14T07:18:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568787365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:47,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568787472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:47,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568787682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:47,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:47,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114894a995481cb4203ac2a1b3c96cbbc27_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114894a995481cb4203ac2a1b3c96cbbc27_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:47,703 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/96bdf8fceb0f4e45974762c4f8719755, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:47,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/96bdf8fceb0f4e45974762c4f8719755 is 175, key is test_row_0/A:col10/1731568726139/Put/seqid=0 2024-11-14T07:18:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742430_1606 (size=39949) 2024-11-14T07:18:47,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/96bdf8fceb0f4e45974762c4f8719755 2024-11-14T07:18:47,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c8c56dbb29bb49009c578b63e59b1f13 is 50, key is test_row_0/B:col10/1731568726139/Put/seqid=0 2024-11-14T07:18:47,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568787712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:47,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742431_1607 (size=12301) 2024-11-14T07:18:47,717 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,33845,1731568550163 for 'test_row_2' in cb55791d898838cbae8eb546c5ba678a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:18:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:18:47,792 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:47,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-14T07:18:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:18:47,794 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:47,795 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:47,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:18:47,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:18:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:47,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568787992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:48,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:18:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:18:48,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c8c56dbb29bb49009c578b63e59b1f13 2024-11-14T07:18:48,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/4309e1c8c92e45d4a607758c39d37dea is 50, key is test_row_0/C:col10/1731568726139/Put/seqid=0 2024-11-14T07:18:48,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742432_1608 (size=12301) 2024-11-14T07:18:48,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:18:48,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:18:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:18:48,498 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:18:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568788501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:48,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/4309e1c8c92e45d4a607758c39d37dea 2024-11-14T07:18:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/96bdf8fceb0f4e45974762c4f8719755 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/96bdf8fceb0f4e45974762c4f8719755 2024-11-14T07:18:48,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/96bdf8fceb0f4e45974762c4f8719755, entries=200, sequenceid=298, filesize=39.0 K 2024-11-14T07:18:48,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c8c56dbb29bb49009c578b63e59b1f13 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c8c56dbb29bb49009c578b63e59b1f13 2024-11-14T07:18:48,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c8c56dbb29bb49009c578b63e59b1f13, entries=150, sequenceid=298, filesize=12.0 K 2024-11-14T07:18:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/4309e1c8c92e45d4a607758c39d37dea as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/4309e1c8c92e45d4a607758c39d37dea 2024-11-14T07:18:48,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/4309e1c8c92e45d4a607758c39d37dea, entries=150, sequenceid=298, filesize=12.0 K 2024-11-14T07:18:48,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cb55791d898838cbae8eb546c5ba678a in 1253ms, sequenceid=298, compaction requested=true 2024-11-14T07:18:48,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:48,538 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:48,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:48,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:48,539 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:48,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:48,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:48,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:48,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:48,539 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:48,539 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/A is initiating minor compaction (all files) 2024-11-14T07:18:48,539 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/A in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,539 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/e64624975a634223babba205e92cef1d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/2a794ca28fb944fda2e558be4af9be56, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/96bdf8fceb0f4e45974762c4f8719755] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=100.4 K 2024-11-14T07:18:48,539 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,540 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/e64624975a634223babba205e92cef1d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/2a794ca28fb944fda2e558be4af9be56, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/96bdf8fceb0f4e45974762c4f8719755] 2024-11-14T07:18:48,540 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:48,540 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/B is initiating minor compaction (all files) 2024-11-14T07:18:48,540 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/B in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,540 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/7f235ee292c449ee8b3b364e87f69db7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/05f20d1f789649418a36e688200b81b0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c8c56dbb29bb49009c578b63e59b1f13] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=36.4 K 2024-11-14T07:18:48,540 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting e64624975a634223babba205e92cef1d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568724273 2024-11-14T07:18:48,540 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f235ee292c449ee8b3b364e87f69db7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568724273 2024-11-14T07:18:48,542 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f20d1f789649418a36e688200b81b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731568725530 2024-11-14T07:18:48,542 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a794ca28fb944fda2e558be4af9be56, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731568725530 2024-11-14T07:18:48,542 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c56dbb29bb49009c578b63e59b1f13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731568726128 2024-11-14T07:18:48,542 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96bdf8fceb0f4e45974762c4f8719755, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731568726128 2024-11-14T07:18:48,550 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:48,551 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/cf514c159ec14b03876dc011eca79e3d is 50, key is test_row_0/B:col10/1731568726139/Put/seqid=0 2024-11-14T07:18:48,557 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:48,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:18:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,558 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T07:18:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:48,564 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411141d8ca6e9f8844b22a48f0a06104c0fdc_cb55791d898838cbae8eb546c5ba678a store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:48,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742433_1609 (size=12949) 2024-11-14T07:18:48,565 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411141d8ca6e9f8844b22a48f0a06104c0fdc_cb55791d898838cbae8eb546c5ba678a, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:48,566 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141d8ca6e9f8844b22a48f0a06104c0fdc_cb55791d898838cbae8eb546c5ba678a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:48,569 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/cf514c159ec14b03876dc011eca79e3d as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/cf514c159ec14b03876dc011eca79e3d 2024-11-14T07:18:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dc77ab40116e44c5aa2932f995acba39_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568727364/Put/seqid=0 2024-11-14T07:18:48,574 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/B of cb55791d898838cbae8eb546c5ba678a into cf514c159ec14b03876dc011eca79e3d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:48,574 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:48,574 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/B, priority=13, startTime=1731568728539; duration=0sec 2024-11-14T07:18:48,574 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:48,574 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:B 2024-11-14T07:18:48,574 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:48,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742434_1610 (size=4469) 2024-11-14T07:18:48,575 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:48,575 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/C is initiating minor compaction (all files) 2024-11-14T07:18:48,576 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/C in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:48,576 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/01fffd694d984a85ae6a9a7eecbd6bac, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9fcf8507fdfd4434bba30e09804015bf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/4309e1c8c92e45d4a607758c39d37dea] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=36.4 K 2024-11-14T07:18:48,576 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#A#compaction#510 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:48,576 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 01fffd694d984a85ae6a9a7eecbd6bac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731568724273 2024-11-14T07:18:48,576 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fcf8507fdfd4434bba30e09804015bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731568725530 2024-11-14T07:18:48,576 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 4309e1c8c92e45d4a607758c39d37dea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731568726128 2024-11-14T07:18:48,576 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/d79b4d49de8b4cbe860ab812205df6f3 is 175, key is test_row_0/A:col10/1731568726139/Put/seqid=0 2024-11-14T07:18:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742435_1611 (size=12454) 2024-11-14T07:18:48,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:48,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dc77ab40116e44c5aa2932f995acba39_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dc77ab40116e44c5aa2932f995acba39_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:48,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/f4b26820a9154ac1b6a949d9969739bf, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/f4b26820a9154ac1b6a949d9969739bf is 175, key is test_row_0/A:col10/1731568727364/Put/seqid=0 2024-11-14T07:18:48,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742436_1612 (size=31903) 2024-11-14T07:18:48,594 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/d79b4d49de8b4cbe860ab812205df6f3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/d79b4d49de8b4cbe860ab812205df6f3 2024-11-14T07:18:48,595 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:48,595 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/ec0c3fb8bc40488aa09f2dcee9d6ab0b is 50, key is test_row_0/C:col10/1731568726139/Put/seqid=0 2024-11-14T07:18:48,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742437_1613 (size=31255) 2024-11-14T07:18:48,601 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/f4b26820a9154ac1b6a949d9969739bf 2024-11-14T07:18:48,601 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/A of cb55791d898838cbae8eb546c5ba678a into d79b4d49de8b4cbe860ab812205df6f3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:48,601 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:48,602 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/A, priority=13, startTime=1731568728538; duration=0sec 2024-11-14T07:18:48,602 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:48,602 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:A 2024-11-14T07:18:48,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/f1941bd038554a47a5f9daabe55c1452 is 50, key is test_row_0/B:col10/1731568727364/Put/seqid=0 2024-11-14T07:18:48,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742438_1614 (size=12949) 2024-11-14T07:18:48,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742439_1615 (size=12301) 2024-11-14T07:18:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:18:49,028 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/ec0c3fb8bc40488aa09f2dcee9d6ab0b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ec0c3fb8bc40488aa09f2dcee9d6ab0b 2024-11-14T07:18:49,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/f1941bd038554a47a5f9daabe55c1452 2024-11-14T07:18:49,031 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/C of cb55791d898838cbae8eb546c5ba678a into ec0c3fb8bc40488aa09f2dcee9d6ab0b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:49,031 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:49,031 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/C, priority=13, startTime=1731568728539; duration=0sec 2024-11-14T07:18:49,031 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:49,031 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:C 2024-11-14T07:18:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/553a61dabf234f6ba088b586b99cc84c is 50, key is test_row_0/C:col10/1731568727364/Put/seqid=0 2024-11-14T07:18:49,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742440_1616 (size=12301) 2024-11-14T07:18:49,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/553a61dabf234f6ba088b586b99cc84c 2024-11-14T07:18:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/f4b26820a9154ac1b6a949d9969739bf as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/f4b26820a9154ac1b6a949d9969739bf 2024-11-14T07:18:49,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/f4b26820a9154ac1b6a949d9969739bf, entries=150, sequenceid=318, filesize=30.5 K 2024-11-14T07:18:49,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/f1941bd038554a47a5f9daabe55c1452 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1941bd038554a47a5f9daabe55c1452 2024-11-14T07:18:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1941bd038554a47a5f9daabe55c1452, entries=150, sequenceid=318, filesize=12.0 K 2024-11-14T07:18:49,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/553a61dabf234f6ba088b586b99cc84c as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/553a61dabf234f6ba088b586b99cc84c 2024-11-14T07:18:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,463 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/553a61dabf234f6ba088b586b99cc84c, entries=150, sequenceid=318, filesize=12.0 K 2024-11-14T07:18:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,463 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for cb55791d898838cbae8eb546c5ba678a in 905ms, sequenceid=318, compaction requested=false 2024-11-14T07:18:49,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:49,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:49,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-14T07:18:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-14T07:18:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-14T07:18:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6700 sec 2024-11-14T07:18:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.6740 sec 2024-11-14T07:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T07:18:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111409586965ce724965bb7805ee8847943d_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_1/A:col10/1731568729615/Put/seqid=0 2024-11-14T07:18:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742441_1617 (size=14994) 2024-11-14T07:18:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,656 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111409586965ce724965bb7805ee8847943d_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409586965ce724965bb7805ee8847943d_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/45a75ea32844416e8c4c374f42b242da, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/45a75ea32844416e8c4c374f42b242da is 175, key is test_row_1/A:col10/1731568729615/Put/seqid=0 2024-11-14T07:18:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742442_1618 (size=39945) 2024-11-14T07:18:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:49,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568729737, max timestamp=1731568729757Num uniques 859; 2024-11-14T07:18:49,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:49,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568729737, max timestamp=1731568729757Num uniques 859; 2024-11-14T07:18:49,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:49,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731568729737, max timestamp=1731568729757Num uniques 859; 2024-11-14T07:18:49,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T07:18:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568729795, max timestamp=1731568729824Num uniques 1018; 2024-11-14T07:18:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T07:18:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568729795, max timestamp=1731568729824Num uniques 1018; 2024-11-14T07:18:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T07:18:49,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731568729795, max timestamp=1731568729824Num uniques 1018; 2024-11-14T07:18:49,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T07:18:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568789908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568789908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:18:49,942 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:49,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:18:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-14T07:18:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:18:49,945 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:18:49,947 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:18:49,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:18:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568790022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568790022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:18:50,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/45a75ea32844416e8c4c374f42b242da 2024-11-14T07:18:50,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:50,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/6f8888da17bc4c5f9106d9e7318113d3 is 50, key is test_row_1/B:col10/1731568729615/Put/seqid=0 2024-11-14T07:18:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:50,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742443_1619 (size=9857) 2024-11-14T07:18:50,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/6f8888da17bc4c5f9106d9e7318113d3 2024-11-14T07:18:50,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6e2b674479e14c4ba5badcb5a58b6a41 is 50, key is test_row_1/C:col10/1731568729615/Put/seqid=0 2024-11-14T07:18:50,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742444_1620 (size=9857) 2024-11-14T07:18:50,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568790232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568790232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:18:50,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:50,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:50,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6e2b674479e14c4ba5badcb5a58b6a41 2024-11-14T07:18:50,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/45a75ea32844416e8c4c374f42b242da as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/45a75ea32844416e8c4c374f42b242da 2024-11-14T07:18:50,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/45a75ea32844416e8c4c374f42b242da, entries=200, sequenceid=331, filesize=39.0 K 2024-11-14T07:18:50,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/6f8888da17bc4c5f9106d9e7318113d3 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/6f8888da17bc4c5f9106d9e7318113d3 2024-11-14T07:18:50,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/6f8888da17bc4c5f9106d9e7318113d3, entries=100, sequenceid=331, filesize=9.6 K 2024-11-14T07:18:50,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/6e2b674479e14c4ba5badcb5a58b6a41 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6e2b674479e14c4ba5badcb5a58b6a41 2024-11-14T07:18:50,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6e2b674479e14c4ba5badcb5a58b6a41, entries=100, sequenceid=331, filesize=9.6 K 2024-11-14T07:18:50,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cb55791d898838cbae8eb546c5ba678a in 915ms, sequenceid=331, compaction requested=true 2024-11-14T07:18:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:18:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:50,531 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:18:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb55791d898838cbae8eb546c5ba678a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:18:50,531 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:50,532 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:50,532 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103103 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:50,532 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/A is initiating minor compaction (all files) 2024-11-14T07:18:50,532 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/B is initiating minor compaction (all files) 2024-11-14T07:18:50,532 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/B in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,532 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/A in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,532 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/cf514c159ec14b03876dc011eca79e3d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1941bd038554a47a5f9daabe55c1452, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/6f8888da17bc4c5f9106d9e7318113d3] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=34.3 K 2024-11-14T07:18:50,532 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/d79b4d49de8b4cbe860ab812205df6f3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/f4b26820a9154ac1b6a949d9969739bf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/45a75ea32844416e8c4c374f42b242da] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=100.7 K 2024-11-14T07:18:50,532 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,532 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. files: [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/d79b4d49de8b4cbe860ab812205df6f3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/f4b26820a9154ac1b6a949d9969739bf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/45a75ea32844416e8c4c374f42b242da] 2024-11-14T07:18:50,533 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting cf514c159ec14b03876dc011eca79e3d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731568726128 2024-11-14T07:18:50,533 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79b4d49de8b4cbe860ab812205df6f3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731568726128 2024-11-14T07:18:50,533 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting f1941bd038554a47a5f9daabe55c1452, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731568727335 2024-11-14T07:18:50,533 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4b26820a9154ac1b6a949d9969739bf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731568727335 2024-11-14T07:18:50,533 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8888da17bc4c5f9106d9e7318113d3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731568729597 2024-11-14T07:18:50,533 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45a75ea32844416e8c4c374f42b242da, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731568729571 2024-11-14T07:18:50,541 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:50,545 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148d0c2e99c61d46ec99e8d409e2088229_cb55791d898838cbae8eb546c5ba678a store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:50,546 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:50,546 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c567bab9db3246619b0e14053c4e7428 is 50, key is test_row_0/B:col10/1731568727364/Put/seqid=0 2024-11-14T07:18:50,546 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148d0c2e99c61d46ec99e8d409e2088229_cb55791d898838cbae8eb546c5ba678a, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:50,547 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148d0c2e99c61d46ec99e8d409e2088229_cb55791d898838cbae8eb546c5ba678a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:50,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=140.89 KB heapSize=399.61 KB 2024-11-14T07:18:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731568729870, max timestamp=1731568730546Num uniques 1077; 2024-11-14T07:18:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c7aa6e2790b3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T07:18:50,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:50,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T07:18:50,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:50,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:18:50,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:50,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T07:18:50,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742445_1621 (size=4469) 2024-11-14T07:18:50,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,568 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#A#compaction#518 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:50,569 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/af2792b9b91e411488d8158bd33fd946 is 175, key is test_row_0/A:col10/1731568727364/Put/seqid=0 2024-11-14T07:18:50,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d0cc546f4ca04923a618c04df5bb3786_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568730546/Put/seqid=0 2024-11-14T07:18:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:18:50,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742446_1622 (size=13051) 2024-11-14T07:18:50,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742447_1623 (size=32112) 2024-11-14T07:18:50,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742448_1624 (size=12454) 2024-11-14T07:18:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568790592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568790597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568790701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568790712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:50,879 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-11-14T07:18:50,879 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:50,879 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,879 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:50,880 INFO [Thread-2390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2390. 2024-11-14T07:18:50,880 DEBUG [Thread-2390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:50,880 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,880 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:50,881 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-11-14T07:18:50,881 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:50,881 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,881 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:50,885 INFO [Thread-2394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2394. 2024-11-14T07:18:50,885 DEBUG [Thread-2394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:50,885 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,885 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:50,885 INFO [Thread-2392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2392. 2024-11-14T07:18:50,885 DEBUG [Thread-2392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:50,885 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,885 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:50,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568790912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568790922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:50,975 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c567bab9db3246619b0e14053c4e7428 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c567bab9db3246619b0e14053c4e7428 2024-11-14T07:18:50,978 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/B of cb55791d898838cbae8eb546c5ba678a into c567bab9db3246619b0e14053c4e7428(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:50,978 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:50,978 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/B, priority=13, startTime=1731568730531; duration=0sec 2024-11-14T07:18:50,978 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:18:50,978 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:B 2024-11-14T07:18:50,979 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:18:50,979 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:18:50,979 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1541): cb55791d898838cbae8eb546c5ba678a/C is initiating minor compaction (all files) 2024-11-14T07:18:50,979 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb55791d898838cbae8eb546c5ba678a/C in TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:50,979 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ec0c3fb8bc40488aa09f2dcee9d6ab0b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/553a61dabf234f6ba088b586b99cc84c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6e2b674479e14c4ba5badcb5a58b6a41] into tmpdir=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp, totalSize=34.3 K 2024-11-14T07:18:50,980 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting ec0c3fb8bc40488aa09f2dcee9d6ab0b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731568726128 2024-11-14T07:18:50,980 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 553a61dabf234f6ba088b586b99cc84c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731568727335 2024-11-14T07:18:50,980 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2b674479e14c4ba5badcb5a58b6a41, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731568729597 2024-11-14T07:18:50,984 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/af2792b9b91e411488d8158bd33fd946 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/af2792b9b91e411488d8158bd33fd946 2024-11-14T07:18:50,985 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb55791d898838cbae8eb546c5ba678a#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:18:50,986 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/24a875b8070f4a828c357ccc62c4512f is 50, key is test_row_0/C:col10/1731568727364/Put/seqid=0 2024-11-14T07:18:50,987 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/A of cb55791d898838cbae8eb546c5ba678a into af2792b9b91e411488d8158bd33fd946(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:50,987 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:50,988 INFO [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/A, priority=13, startTime=1731568730531; duration=0sec 2024-11-14T07:18:50,988 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:50,988 DEBUG [RS:0;c7aa6e2790b3:33845-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:A 2024-11-14T07:18:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742449_1625 (size=13051) 2024-11-14T07:18:50,989 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:50,992 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d0cc546f4ca04923a618c04df5bb3786_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d0cc546f4ca04923a618c04df5bb3786_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:50,992 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/a24c7849d8bb4d179bc704e032fdc147, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:50,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/a24c7849d8bb4d179bc704e032fdc147 is 175, key is test_row_0/A:col10/1731568730546/Put/seqid=0 2024-11-14T07:18:50,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742450_1626 (size=31255) 2024-11-14T07:18:51,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:18:51,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568791221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568791232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:51,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,392 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/24a875b8070f4a828c357ccc62c4512f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/24a875b8070f4a828c357ccc62c4512f 2024-11-14T07:18:51,395 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb55791d898838cbae8eb546c5ba678a/C of cb55791d898838cbae8eb546c5ba678a into 24a875b8070f4a828c357ccc62c4512f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:18:51,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/a24c7849d8bb4d179bc704e032fdc147 2024-11-14T07:18:51,395 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:51,396 INFO [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., storeName=cb55791d898838cbae8eb546c5ba678a/C, priority=13, startTime=1731568730531; duration=0sec 2024-11-14T07:18:51,396 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:18:51,396 DEBUG [RS:0;c7aa6e2790b3:33845-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb55791d898838cbae8eb546c5ba678a:C 2024-11-14T07:18:51,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/b37b51f6fbc54c7e922124cbe900ee4b is 50, key is test_row_0/B:col10/1731568730546/Put/seqid=0 2024-11-14T07:18:51,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742451_1627 (size=12301) 2024-11-14T07:18:51,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/b37b51f6fbc54c7e922124cbe900ee4b 2024-11-14T07:18:51,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/0030845b94294e55b64f7b5b3f9438c0 is 50, key is test_row_0/C:col10/1731568730546/Put/seqid=0 2024-11-14T07:18:51,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742452_1628 (size=12301) 2024-11-14T07:18:51,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:51,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731568791732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:18:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38632 deadline: 1731568791742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 , the old value is region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:18:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., hostname=c7aa6e2790b3,33845,1731568550163, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:18:51,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:18:51,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/0030845b94294e55b64f7b5b3f9438c0 2024-11-14T07:18:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/a24c7849d8bb4d179bc704e032fdc147 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/a24c7849d8bb4d179bc704e032fdc147 2024-11-14T07:18:51,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/a24c7849d8bb4d179bc704e032fdc147, entries=150, sequenceid=386, filesize=30.5 K 2024-11-14T07:18:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/b37b51f6fbc54c7e922124cbe900ee4b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b37b51f6fbc54c7e922124cbe900ee4b 2024-11-14T07:18:51,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b37b51f6fbc54c7e922124cbe900ee4b, entries=150, sequenceid=386, filesize=12.0 K 2024-11-14T07:18:51,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/0030845b94294e55b64f7b5b3f9438c0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/0030845b94294e55b64f7b5b3f9438c0 2024-11-14T07:18:51,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/0030845b94294e55b64f7b5b3f9438c0, entries=150, sequenceid=386, filesize=12.0 K 2024-11-14T07:18:51,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~306.55 KB/313904, currentSize=73.80 KB/75570 for cb55791d898838cbae8eb546c5ba678a in 1274ms, sequenceid=386, compaction requested=false 2024-11-14T07:18:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:51,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:18:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:51,933 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T07:18:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:51,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aba33f1c8758460195b1d98b1fc2d1dc_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568730590/Put/seqid=0 2024-11-14T07:18:51,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742453_1629 (size=12454) 2024-11-14T07:18:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:18:52,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:52,344 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aba33f1c8758460195b1d98b1fc2d1dc_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aba33f1c8758460195b1d98b1fc2d1dc_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:52,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/b6182b21c69846fb9e8fc6dde5439efb, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:52,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/b6182b21c69846fb9e8fc6dde5439efb is 175, key is test_row_0/A:col10/1731568730590/Put/seqid=0 2024-11-14T07:18:52,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742454_1630 (size=31255) 2024-11-14T07:18:52,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. as already flushing 2024-11-14T07:18:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33845 {}] regionserver.HRegion(8855): Flush requested on cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:52,743 INFO [Thread-2385 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2385. 2024-11-14T07:18:52,743 DEBUG [Thread-2385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:52,743 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:52,743 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:52,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:52,749 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/b6182b21c69846fb9e8fc6dde5439efb 2024-11-14T07:18:52,753 INFO [Thread-2383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2383. 2024-11-14T07:18:52,753 DEBUG [Thread-2383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:52,753 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:52,753 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:52,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:52,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/fe2e93fb3d894a22a3073daeb0424f21 is 50, key is test_row_0/B:col10/1731568730590/Put/seqid=0 2024-11-14T07:18:52,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742455_1631 (size=12301) 2024-11-14T07:18:53,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/fe2e93fb3d894a22a3073daeb0424f21 2024-11-14T07:18:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9681c8516cdc4cd3bb1e2278faab6210 is 50, key is test_row_0/C:col10/1731568730590/Put/seqid=0 2024-11-14T07:18:53,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742456_1632 (size=12301) 2024-11-14T07:18:53,565 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9681c8516cdc4cd3bb1e2278faab6210 2024-11-14T07:18:53,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/b6182b21c69846fb9e8fc6dde5439efb as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/b6182b21c69846fb9e8fc6dde5439efb 2024-11-14T07:18:53,571 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/b6182b21c69846fb9e8fc6dde5439efb, entries=150, sequenceid=403, filesize=30.5 K 2024-11-14T07:18:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/fe2e93fb3d894a22a3073daeb0424f21 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/fe2e93fb3d894a22a3073daeb0424f21 2024-11-14T07:18:53,574 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/fe2e93fb3d894a22a3073daeb0424f21, entries=150, sequenceid=403, filesize=12.0 K 2024-11-14T07:18:53,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/9681c8516cdc4cd3bb1e2278faab6210 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9681c8516cdc4cd3bb1e2278faab6210 2024-11-14T07:18:53,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9681c8516cdc4cd3bb1e2278faab6210, entries=150, sequenceid=403, filesize=12.0 K 2024-11-14T07:18:53,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for cb55791d898838cbae8eb546c5ba678a in 1645ms, sequenceid=403, compaction requested=true 2024-11-14T07:18:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for cb55791d898838cbae8eb546c5ba678a: 2024-11-14T07:18:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-14T07:18:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-14T07:18:53,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-14T07:18:53,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6320 sec 2024-11-14T07:18:53,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.6370 sec 2024-11-14T07:18:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:18:54,102 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:55,713 INFO [Thread-2379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2379. 2024-11-14T07:18:55,713 DEBUG [Thread-2379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:55,713 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,713 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:55,723 INFO [Thread-2387 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2387. 2024-11-14T07:18:55,723 DEBUG [Thread-2387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:55,723 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,723 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:55,743 INFO [Thread-2381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2381. 2024-11-14T07:18:55,743 DEBUG [Thread-2381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:18:55,743 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,744 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3485 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3327 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3311 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3486 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3331 2024-11-14T07:18:55,744 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:18:55,744 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:18:55,744 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:18:55,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:55,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:55,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:18:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:18:55,747 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568735747"}]},"ts":"1731568735747"} 2024-11-14T07:18:55,748 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:18:55,748 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:18:55,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:18:55,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, UNASSIGN}] 2024-11-14T07:18:55,750 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, UNASSIGN 2024-11-14T07:18:55,751 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=CLOSING, regionLocation=c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:55,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, UNASSIGN because future has completed 2024-11-14T07:18:55,752 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:18:55,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163}] 2024-11-14T07:18:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:18:55,904 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:55,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:18:55,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing cb55791d898838cbae8eb546c5ba678a, disabling compactions & flushes 2024-11-14T07:18:55,904 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:55,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:55,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. after waiting 0 ms 2024-11-14T07:18:55,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:55,904 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing cb55791d898838cbae8eb546c5ba678a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T07:18:55,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=A 2024-11-14T07:18:55,904 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:55,905 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=B 2024-11-14T07:18:55,905 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:55,905 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb55791d898838cbae8eb546c5ba678a, store=C 2024-11-14T07:18:55,905 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T07:18:55,909 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b62f4bbb7b04b37ba1d2e4de9e74773_cb55791d898838cbae8eb546c5ba678a is 50, key is test_row_0/A:col10/1731568735742/Put/seqid=0 2024-11-14T07:18:55,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742457_1633 (size=9914) 2024-11-14T07:18:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:18:56,312 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:18:56,315 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b62f4bbb7b04b37ba1d2e4de9e74773_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b62f4bbb7b04b37ba1d2e4de9e74773_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:56,315 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8333402aca8d4eeaac34da23e5856d4b, store: [table=TestAcidGuarantees family=A region=cb55791d898838cbae8eb546c5ba678a] 2024-11-14T07:18:56,316 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8333402aca8d4eeaac34da23e5856d4b is 175, key is test_row_0/A:col10/1731568735742/Put/seqid=0 2024-11-14T07:18:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742458_1634 (size=22561) 2024-11-14T07:18:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:18:56,719 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8333402aca8d4eeaac34da23e5856d4b 2024-11-14T07:18:56,724 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c14b85372ef04ad5b2cdd510566ee685 is 50, key is test_row_0/B:col10/1731568735742/Put/seqid=0 2024-11-14T07:18:56,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742459_1635 (size=9857) 2024-11-14T07:18:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:18:57,127 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c14b85372ef04ad5b2cdd510566ee685 2024-11-14T07:18:57,132 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/7570fccc94614b29ac9d1a5841464964 is 50, key is test_row_0/C:col10/1731568735742/Put/seqid=0 2024-11-14T07:18:57,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742460_1636 (size=9857) 2024-11-14T07:18:57,535 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/7570fccc94614b29ac9d1a5841464964 2024-11-14T07:18:57,538 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/A/8333402aca8d4eeaac34da23e5856d4b as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8333402aca8d4eeaac34da23e5856d4b 2024-11-14T07:18:57,541 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8333402aca8d4eeaac34da23e5856d4b, entries=100, sequenceid=411, filesize=22.0 K 2024-11-14T07:18:57,542 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/B/c14b85372ef04ad5b2cdd510566ee685 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c14b85372ef04ad5b2cdd510566ee685 2024-11-14T07:18:57,545 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c14b85372ef04ad5b2cdd510566ee685, entries=100, sequenceid=411, filesize=9.6 K 2024-11-14T07:18:57,545 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/.tmp/C/7570fccc94614b29ac9d1a5841464964 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/7570fccc94614b29ac9d1a5841464964 2024-11-14T07:18:57,548 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/7570fccc94614b29ac9d1a5841464964, entries=100, sequenceid=411, filesize=9.6 K 2024-11-14T07:18:57,549 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cb55791d898838cbae8eb546c5ba678a in 1644ms, sequenceid=411, compaction requested=true 2024-11-14T07:18:57,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/5d3b80f9185f4e0ba70b6fce653cc5f2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1fe276f460ed453ca5846a195a097bc3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/639b2e6f81ce4f449f4390d9df67fc51, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/13cc0517ae2d41b98bda10b42279dc58, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/eccbb94d5a2348a7b92442eb40a0eee8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/ce831ff4d6534d63b59c748103a720b1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1a05520483404160abe0bfa20baf1904, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/832413b43fe14e3da41af8f48d822d3a, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/dbb3d3483d1c4dacadaf90e15f0467db, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/9d9a5d9b061541598dac1d12a4707dc6, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/09fdb0ce0a6f494baf36214def9b5428, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/92781d7116534c2b8fb8978f27819d52, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/630654e609934d62bfc76590ed6a2b0e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6f58a91c060f4e7587efbbd4955754fb, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6cf6a775434d4ba49c959197f66732d8, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8a968c3816ad4bf6b696d1faca0da403, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/e64624975a634223babba205e92cef1d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/2a794ca28fb944fda2e558be4af9be56, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/96bdf8fceb0f4e45974762c4f8719755, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/d79b4d49de8b4cbe860ab812205df6f3, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/f4b26820a9154ac1b6a949d9969739bf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/45a75ea32844416e8c4c374f42b242da] to archive 2024-11-14T07:18:57,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:18:57,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/5d3b80f9185f4e0ba70b6fce653cc5f2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/5d3b80f9185f4e0ba70b6fce653cc5f2 2024-11-14T07:18:57,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1fe276f460ed453ca5846a195a097bc3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1fe276f460ed453ca5846a195a097bc3 2024-11-14T07:18:57,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/639b2e6f81ce4f449f4390d9df67fc51 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/639b2e6f81ce4f449f4390d9df67fc51 2024-11-14T07:18:57,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/13cc0517ae2d41b98bda10b42279dc58 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/13cc0517ae2d41b98bda10b42279dc58 2024-11-14T07:18:57,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/eccbb94d5a2348a7b92442eb40a0eee8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/eccbb94d5a2348a7b92442eb40a0eee8 2024-11-14T07:18:57,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/ce831ff4d6534d63b59c748103a720b1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/ce831ff4d6534d63b59c748103a720b1 2024-11-14T07:18:57,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1a05520483404160abe0bfa20baf1904 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/1a05520483404160abe0bfa20baf1904 2024-11-14T07:18:57,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/832413b43fe14e3da41af8f48d822d3a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/832413b43fe14e3da41af8f48d822d3a 2024-11-14T07:18:57,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/dbb3d3483d1c4dacadaf90e15f0467db to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/dbb3d3483d1c4dacadaf90e15f0467db 2024-11-14T07:18:57,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/9d9a5d9b061541598dac1d12a4707dc6 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/9d9a5d9b061541598dac1d12a4707dc6 2024-11-14T07:18:57,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/09fdb0ce0a6f494baf36214def9b5428 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/09fdb0ce0a6f494baf36214def9b5428 2024-11-14T07:18:57,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/92781d7116534c2b8fb8978f27819d52 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/92781d7116534c2b8fb8978f27819d52 2024-11-14T07:18:57,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/630654e609934d62bfc76590ed6a2b0e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/630654e609934d62bfc76590ed6a2b0e 2024-11-14T07:18:57,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6f58a91c060f4e7587efbbd4955754fb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6f58a91c060f4e7587efbbd4955754fb 2024-11-14T07:18:57,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6cf6a775434d4ba49c959197f66732d8 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/6cf6a775434d4ba49c959197f66732d8 2024-11-14T07:18:57,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8a968c3816ad4bf6b696d1faca0da403 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8a968c3816ad4bf6b696d1faca0da403 2024-11-14T07:18:57,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/e64624975a634223babba205e92cef1d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/e64624975a634223babba205e92cef1d 2024-11-14T07:18:57,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/2a794ca28fb944fda2e558be4af9be56 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/2a794ca28fb944fda2e558be4af9be56 2024-11-14T07:18:57,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/96bdf8fceb0f4e45974762c4f8719755 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/96bdf8fceb0f4e45974762c4f8719755 2024-11-14T07:18:57,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/d79b4d49de8b4cbe860ab812205df6f3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/d79b4d49de8b4cbe860ab812205df6f3 2024-11-14T07:18:57,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/f4b26820a9154ac1b6a949d9969739bf to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/f4b26820a9154ac1b6a949d9969739bf 2024-11-14T07:18:57,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/45a75ea32844416e8c4c374f42b242da to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/45a75ea32844416e8c4c374f42b242da 2024-11-14T07:18:57,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa60a268eafe482c806744b1d5074b9e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8518b4a8f3c74e5c9448fd4f2d160bfe, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/9ccd02bf16d04aa79f75a2b0f5371329, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1c7ef4daad14d77b70bf69072a7037b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/ab7c3aa37a7747459427da8eae86707e, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/4f9b2aac441240ff8ea1fa406dda95d7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/41d49234722145df89e2f645e611b060, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8178d8fdcde54eecbf244dfcbbee9921, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/93ffce38829540539be2d90e1255fa3b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/baaf94e08f2e4513980d8fe1cc6b8aae, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa7666d2d1644004a14590cbee7716d4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/d0f97deb757f45cfa3ae434e52a3146d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b07fc80cf96d4914a4696e640be18163, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/797b84bb9c3144eca8be71c0f4ca7a50, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/e54d30f49e2f4099bc8109691f636ae1, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/7f235ee292c449ee8b3b364e87f69db7, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/19a5998101b446adac97a48289644a56, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/05f20d1f789649418a36e688200b81b0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/cf514c159ec14b03876dc011eca79e3d, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c8c56dbb29bb49009c578b63e59b1f13, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1941bd038554a47a5f9daabe55c1452, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/6f8888da17bc4c5f9106d9e7318113d3] to archive 2024-11-14T07:18:57,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:18:57,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa60a268eafe482c806744b1d5074b9e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa60a268eafe482c806744b1d5074b9e 2024-11-14T07:18:57,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8518b4a8f3c74e5c9448fd4f2d160bfe to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8518b4a8f3c74e5c9448fd4f2d160bfe 2024-11-14T07:18:57,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/9ccd02bf16d04aa79f75a2b0f5371329 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/9ccd02bf16d04aa79f75a2b0f5371329 2024-11-14T07:18:57,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1c7ef4daad14d77b70bf69072a7037b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1c7ef4daad14d77b70bf69072a7037b 2024-11-14T07:18:57,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/ab7c3aa37a7747459427da8eae86707e to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/ab7c3aa37a7747459427da8eae86707e 2024-11-14T07:18:57,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/4f9b2aac441240ff8ea1fa406dda95d7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/4f9b2aac441240ff8ea1fa406dda95d7 2024-11-14T07:18:57,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/41d49234722145df89e2f645e611b060 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/41d49234722145df89e2f645e611b060 2024-11-14T07:18:57,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8178d8fdcde54eecbf244dfcbbee9921 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/8178d8fdcde54eecbf244dfcbbee9921 2024-11-14T07:18:57,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/93ffce38829540539be2d90e1255fa3b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/93ffce38829540539be2d90e1255fa3b 2024-11-14T07:18:57,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/baaf94e08f2e4513980d8fe1cc6b8aae to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/baaf94e08f2e4513980d8fe1cc6b8aae 2024-11-14T07:18:57,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa7666d2d1644004a14590cbee7716d4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/aa7666d2d1644004a14590cbee7716d4 2024-11-14T07:18:57,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/d0f97deb757f45cfa3ae434e52a3146d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/d0f97deb757f45cfa3ae434e52a3146d 2024-11-14T07:18:57,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b07fc80cf96d4914a4696e640be18163 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b07fc80cf96d4914a4696e640be18163 2024-11-14T07:18:57,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/797b84bb9c3144eca8be71c0f4ca7a50 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/797b84bb9c3144eca8be71c0f4ca7a50 2024-11-14T07:18:57,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/e54d30f49e2f4099bc8109691f636ae1 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/e54d30f49e2f4099bc8109691f636ae1 2024-11-14T07:18:57,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/7f235ee292c449ee8b3b364e87f69db7 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/7f235ee292c449ee8b3b364e87f69db7 2024-11-14T07:18:57,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/19a5998101b446adac97a48289644a56 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/19a5998101b446adac97a48289644a56 2024-11-14T07:18:57,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/05f20d1f789649418a36e688200b81b0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/05f20d1f789649418a36e688200b81b0 2024-11-14T07:18:57,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/cf514c159ec14b03876dc011eca79e3d to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/cf514c159ec14b03876dc011eca79e3d 2024-11-14T07:18:57,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c8c56dbb29bb49009c578b63e59b1f13 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c8c56dbb29bb49009c578b63e59b1f13 2024-11-14T07:18:57,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1941bd038554a47a5f9daabe55c1452 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/f1941bd038554a47a5f9daabe55c1452 2024-11-14T07:18:57,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/6f8888da17bc4c5f9106d9e7318113d3 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/6f8888da17bc4c5f9106d9e7318113d3 2024-11-14T07:18:57,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/bac2bbd4ce6445549a98030c3b6980e4, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/17c667c652874e95a2275621df4aee6f, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f41b62a0dac943e4a831f8efa76c6aa0, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ab05c2ed37304b29bb0eb527b7231621, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f598a2cd3fea42a6956610dbadecab53, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/21ae655beccc40069c69e3277fe44009, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/927020fd7b984760acbe862d523fab1b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/67956568b9864a28bf2bb00686061b76, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6257472c077f4138b86ac27bd4ce7834, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6f1b8c4e1ffb45eea25cedae37417267, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/161c0afed8e648ac9b826a92d1be6121, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9a95c7c4f4074db08521c0b222f67f0c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/08e20a5ebfbf4073a1e492cddd9420f2, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/42b2f57a85654e0ba1aa18641fc3d797, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/3e06261989ed4028aac0ac4b74598b42, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/01fffd694d984a85ae6a9a7eecbd6bac, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/c6a9da62c0b44d2a8b1efef57c8bfbed, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9fcf8507fdfd4434bba30e09804015bf, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ec0c3fb8bc40488aa09f2dcee9d6ab0b, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/4309e1c8c92e45d4a607758c39d37dea, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/553a61dabf234f6ba088b586b99cc84c, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6e2b674479e14c4ba5badcb5a58b6a41] to archive 2024-11-14T07:18:57,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:18:57,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/bac2bbd4ce6445549a98030c3b6980e4 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/bac2bbd4ce6445549a98030c3b6980e4 2024-11-14T07:18:57,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/17c667c652874e95a2275621df4aee6f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/17c667c652874e95a2275621df4aee6f 2024-11-14T07:18:57,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f41b62a0dac943e4a831f8efa76c6aa0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f41b62a0dac943e4a831f8efa76c6aa0 2024-11-14T07:18:57,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ab05c2ed37304b29bb0eb527b7231621 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ab05c2ed37304b29bb0eb527b7231621 2024-11-14T07:18:57,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f598a2cd3fea42a6956610dbadecab53 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/f598a2cd3fea42a6956610dbadecab53 2024-11-14T07:18:57,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/21ae655beccc40069c69e3277fe44009 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/21ae655beccc40069c69e3277fe44009 2024-11-14T07:18:57,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/927020fd7b984760acbe862d523fab1b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/927020fd7b984760acbe862d523fab1b 2024-11-14T07:18:57,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/67956568b9864a28bf2bb00686061b76 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/67956568b9864a28bf2bb00686061b76 2024-11-14T07:18:57,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6257472c077f4138b86ac27bd4ce7834 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6257472c077f4138b86ac27bd4ce7834 2024-11-14T07:18:57,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6f1b8c4e1ffb45eea25cedae37417267 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6f1b8c4e1ffb45eea25cedae37417267 2024-11-14T07:18:57,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/161c0afed8e648ac9b826a92d1be6121 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/161c0afed8e648ac9b826a92d1be6121 2024-11-14T07:18:57,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9a95c7c4f4074db08521c0b222f67f0c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9a95c7c4f4074db08521c0b222f67f0c 2024-11-14T07:18:57,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/08e20a5ebfbf4073a1e492cddd9420f2 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/08e20a5ebfbf4073a1e492cddd9420f2 2024-11-14T07:18:57,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/42b2f57a85654e0ba1aa18641fc3d797 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/42b2f57a85654e0ba1aa18641fc3d797 2024-11-14T07:18:57,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/3e06261989ed4028aac0ac4b74598b42 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/3e06261989ed4028aac0ac4b74598b42 2024-11-14T07:18:57,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/01fffd694d984a85ae6a9a7eecbd6bac to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/01fffd694d984a85ae6a9a7eecbd6bac 2024-11-14T07:18:57,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/c6a9da62c0b44d2a8b1efef57c8bfbed to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/c6a9da62c0b44d2a8b1efef57c8bfbed 2024-11-14T07:18:57,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9fcf8507fdfd4434bba30e09804015bf to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9fcf8507fdfd4434bba30e09804015bf 2024-11-14T07:18:57,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ec0c3fb8bc40488aa09f2dcee9d6ab0b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/ec0c3fb8bc40488aa09f2dcee9d6ab0b 2024-11-14T07:18:57,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/4309e1c8c92e45d4a607758c39d37dea to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/4309e1c8c92e45d4a607758c39d37dea 2024-11-14T07:18:57,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/553a61dabf234f6ba088b586b99cc84c to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/553a61dabf234f6ba088b586b99cc84c 2024-11-14T07:18:57,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6e2b674479e14c4ba5badcb5a58b6a41 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/6e2b674479e14c4ba5badcb5a58b6a41 2024-11-14T07:18:57,602 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/recovered.edits/414.seqid, newMaxSeqId=414, maxSeqId=4 2024-11-14T07:18:57,602 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. 2024-11-14T07:18:57,602 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for cb55791d898838cbae8eb546c5ba678a: Waiting for close lock at 1731568735904Running coprocessor pre-close hooks at 1731568735904Disabling compacts and flushes for region at 1731568735904Disabling writes for close at 1731568735904Obtaining lock to block concurrent updates at 1731568735904Preparing flush snapshotting stores in cb55791d898838cbae8eb546c5ba678a at 1731568735904Finished memstore snapshotting TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731568735905 (+1 ms)Flushing stores of TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a. at 1731568735905Flushing cb55791d898838cbae8eb546c5ba678a/A: creating writer at 1731568735905Flushing mob file cb55791d898838cbae8eb546c5ba678a/A: appending metadata at 1731568735909 (+4 ms)Flushing mob file cb55791d898838cbae8eb546c5ba678a/A: closing flushed file at 1731568735909Flushing cb55791d898838cbae8eb546c5ba678a/A: appending metadata at 1731568736315 (+406 ms)Flushing cb55791d898838cbae8eb546c5ba678a/A: closing flushed file at 1731568736315Flushing cb55791d898838cbae8eb546c5ba678a/B: creating writer at 1731568736722 (+407 ms)Flushing cb55791d898838cbae8eb546c5ba678a/B: appending metadata at 1731568736723 (+1 ms)Flushing cb55791d898838cbae8eb546c5ba678a/B: closing flushed file at 1731568736723Flushing cb55791d898838cbae8eb546c5ba678a/C: creating writer at 1731568737130 (+407 ms)Flushing cb55791d898838cbae8eb546c5ba678a/C: appending metadata at 1731568737131 (+1 ms)Flushing cb55791d898838cbae8eb546c5ba678a/C: closing flushed file at 1731568737131Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d9181ca: reopening flushed file at 1731568737538 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d624f7f: reopening flushed file at 1731568737541 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a946ec2: reopening flushed file at 1731568737545 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cb55791d898838cbae8eb546c5ba678a in 1644ms, sequenceid=411, compaction requested=true at 1731568737549 (+4 ms)Writing region close event to WAL at 1731568737600 (+51 ms)Running coprocessor post-close hooks at 1731568737602 (+2 ms)Closed at 1731568737602 2024-11-14T07:18:57,604 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,604 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=cb55791d898838cbae8eb546c5ba678a, regionState=CLOSED 2024-11-14T07:18:57,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 because future has completed 2024-11-14T07:18:57,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-14T07:18:57,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure cb55791d898838cbae8eb546c5ba678a, server=c7aa6e2790b3,33845,1731568550163 in 1.8540 sec 2024-11-14T07:18:57,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-14T07:18:57,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb55791d898838cbae8eb546c5ba678a, UNASSIGN in 1.8580 sec 2024-11-14T07:18:57,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-14T07:18:57,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8600 sec 2024-11-14T07:18:57,610 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731568737610"}]},"ts":"1731568737610"} 2024-11-14T07:18:57,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:18:57,611 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:18:57,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-14T07:18:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:18:57,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:57,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:18:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:57,894 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:18:57,895 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:57,896 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,898 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C, FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/recovered.edits] 2024-11-14T07:18:57,900 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8333402aca8d4eeaac34da23e5856d4b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/8333402aca8d4eeaac34da23e5856d4b 2024-11-14T07:18:57,901 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/a24c7849d8bb4d179bc704e032fdc147 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/a24c7849d8bb4d179bc704e032fdc147 2024-11-14T07:18:57,901 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/af2792b9b91e411488d8158bd33fd946 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/af2792b9b91e411488d8158bd33fd946 2024-11-14T07:18:57,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/b6182b21c69846fb9e8fc6dde5439efb to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/A/b6182b21c69846fb9e8fc6dde5439efb 2024-11-14T07:18:57,903 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b37b51f6fbc54c7e922124cbe900ee4b to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/b37b51f6fbc54c7e922124cbe900ee4b 2024-11-14T07:18:57,904 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c14b85372ef04ad5b2cdd510566ee685 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c14b85372ef04ad5b2cdd510566ee685 2024-11-14T07:18:57,904 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c567bab9db3246619b0e14053c4e7428 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/c567bab9db3246619b0e14053c4e7428 2024-11-14T07:18:57,905 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/fe2e93fb3d894a22a3073daeb0424f21 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/B/fe2e93fb3d894a22a3073daeb0424f21 2024-11-14T07:18:57,906 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/0030845b94294e55b64f7b5b3f9438c0 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/0030845b94294e55b64f7b5b3f9438c0 2024-11-14T07:18:57,906 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/24a875b8070f4a828c357ccc62c4512f to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/24a875b8070f4a828c357ccc62c4512f 2024-11-14T07:18:57,907 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/7570fccc94614b29ac9d1a5841464964 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/7570fccc94614b29ac9d1a5841464964 2024-11-14T07:18:57,908 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9681c8516cdc4cd3bb1e2278faab6210 to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/C/9681c8516cdc4cd3bb1e2278faab6210 2024-11-14T07:18:57,909 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/recovered.edits/414.seqid to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a/recovered.edits/414.seqid 2024-11-14T07:18:57,909 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/default/TestAcidGuarantees/cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,909 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:18:57,910 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:18:57,910 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T07:18:57,912 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409586965ce724965bb7805ee8847943d_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409586965ce724965bb7805ee8847943d_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,912 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143fe5101c353f4e449d31fbfbcb85b019_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143fe5101c353f4e449d31fbfbcb85b019_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,913 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b593d36a9c94232a6f7b6bad2b95ff0_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b593d36a9c94232a6f7b6bad2b95ff0_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,913 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b62f4bbb7b04b37ba1d2e4de9e74773_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b62f4bbb7b04b37ba1d2e4de9e74773_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,914 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147d40a927e5c4486b8376e429545ca7cf_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147d40a927e5c4486b8376e429545ca7cf_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,915 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114867aa92ac16d43b9ba6854b0be6420c0_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114867aa92ac16d43b9ba6854b0be6420c0_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,915 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486f0e15fe2f0409383002116d5e3fac9_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486f0e15fe2f0409383002116d5e3fac9_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,916 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114894a995481cb4203ac2a1b3c96cbbc27_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114894a995481cb4203ac2a1b3c96cbbc27_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,917 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148e10b96c24ff4eb89f1ee650e8b3f2a1_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148e10b96c24ff4eb89f1ee650e8b3f2a1_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,917 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cbda0f3e5b0465c8e9a5b8838824061_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cbda0f3e5b0465c8e9a5b8838824061_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,918 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aba33f1c8758460195b1d98b1fc2d1dc_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aba33f1c8758460195b1d98b1fc2d1dc_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,918 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1bd98f49bf143feb34962e72620d5e1_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1bd98f49bf143feb34962e72620d5e1_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,919 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3dbe25a48024cf980854dfdf329ac2e_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3dbe25a48024cf980854dfdf329ac2e_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,920 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d0cc546f4ca04923a618c04df5bb3786_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d0cc546f4ca04923a618c04df5bb3786_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,920 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d32798bef449447881bce06641b411ad_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d32798bef449447881bce06641b411ad_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,921 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d40520d14bce49f2badaa439fdff8a70_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d40520d14bce49f2badaa439fdff8a70_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,922 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d4f1e833c15d4589ba632c3b55f02680_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d4f1e833c15d4589ba632c3b55f02680_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,922 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dc77ab40116e44c5aa2932f995acba39_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dc77ab40116e44c5aa2932f995acba39_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,923 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fbec4c11f57749a1af606cc43a470790_cb55791d898838cbae8eb546c5ba678a to hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fbec4c11f57749a1af606cc43a470790_cb55791d898838cbae8eb546c5ba678a 2024-11-14T07:18:57,923 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:18:57,924 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:57,926 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:18:57,928 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:18:57,929 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:57,929 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:18:57,929 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731568737929"}]},"ts":"9223372036854775807"} 2024-11-14T07:18:57,930 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:18:57,930 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cb55791d898838cbae8eb546c5ba678a, NAME => 'TestAcidGuarantees,,1731568707728.cb55791d898838cbae8eb546c5ba678a.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:18:57,930 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:18:57,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731568737930"}]},"ts":"9223372036854775807"} 2024-11-14T07:18:57,931 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:18:57,931 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:18:57,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-14T07:18:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:18:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:18:58,002 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:18:58,011 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 424), ProcessCount=11 (was 11), AvailableMemoryMB=5355 (was 5385) 2024-11-14T07:18:58,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T07:18:58,011 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:18:58,011 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:18:58,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:58,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:58,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:18:58,012 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T07:18:58,012 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-14T07:18:58,012 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c7aa6e2790b3,37073,1731568549456 2024-11-14T07:18:58,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T07:18:58,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T07:18:58,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:18:58,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:18:58,063 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T07:18:58,064 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:18:58,064 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:18:58,064 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:18:58,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:58,064 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:18:58,065 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c7aa6e2790b3,33845,1731568550163' ***** 2024-11-14T07:18:58,065 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T07:18:58,065 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T07:18:58,065 INFO [RS:0;c7aa6e2790b3:33845 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T07:18:58,065 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T07:18:58,065 INFO [RS:0;c7aa6e2790b3:33845 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T07:18:58,066 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(959): stopping server c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:58,066 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T07:18:58,066 INFO [RS:0;c7aa6e2790b3:33845 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c7aa6e2790b3:33845. 2024-11-14T07:18:58,066 DEBUG [RS:0;c7aa6e2790b3:33845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:18:58,066 DEBUG [RS:0;c7aa6e2790b3:33845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:58,066 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T07:18:58,066 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T07:18:58,066 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T07:18:58,066 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T07:18:58,067 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T07:18:58,067 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T07:18:58,067 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T07:18:58,067 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T07:18:58,067 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T07:18:58,067 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T07:18:58,067 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:58,067 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T07:18:58,068 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T07:18:58,087 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/info/b9e98f391bff44fb9229e3c165baa0e0 is 95, key is TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03./info:/1731568581667/DeleteFamily/seqid=0 2024-11-14T07:18:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742461_1637 (size=6514) 2024-11-14T07:18:58,140 INFO [regionserver/c7aa6e2790b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T07:18:58,268 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:58,468 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:58,490 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/info/b9e98f391bff44fb9229e3c165baa0e0 2024-11-14T07:18:58,510 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/ns/2b8ae52a1b214c1c8949e78d0d7d1aae is 93, key is TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03./ns:/1731568581627/DeleteFamily/seqid=0 2024-11-14T07:18:58,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742462_1638 (size=5906) 2024-11-14T07:18:58,630 INFO [regionserver/c7aa6e2790b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T07:18:58,630 INFO [regionserver/c7aa6e2790b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T07:18:58,668 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:58,868 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:58,913 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/ns/2b8ae52a1b214c1c8949e78d0d7d1aae 2024-11-14T07:18:58,933 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/rep_barrier/4d4629b0ae594c5095e598d022be2e0f is 102, key is TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03./rep_barrier:/1731568581627/DeleteFamily/seqid=0 2024-11-14T07:18:58,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742463_1639 (size=6025) 2024-11-14T07:18:59,068 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:59,268 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:59,337 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/rep_barrier/4d4629b0ae594c5095e598d022be2e0f 2024-11-14T07:18:59,351 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/table/0d6bea0f66f247c3ba3cf25d7f28d936 is 96, key is TestAcidGuarantees,,1731568553894.349b5cba99ebf7c21762f59b06a8ec03./table:/1731568581627/DeleteFamily/seqid=0 2024-11-14T07:18:59,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742464_1640 (size=5936) 2024-11-14T07:18:59,469 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:59,669 DEBUG [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:18:59,755 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/table/0d6bea0f66f247c3ba3cf25d7f28d936 2024-11-14T07:18:59,758 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/info/b9e98f391bff44fb9229e3c165baa0e0 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/info/b9e98f391bff44fb9229e3c165baa0e0 2024-11-14T07:18:59,760 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/info/b9e98f391bff44fb9229e3c165baa0e0, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T07:18:59,761 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/ns/2b8ae52a1b214c1c8949e78d0d7d1aae as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/ns/2b8ae52a1b214c1c8949e78d0d7d1aae 2024-11-14T07:18:59,763 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/ns/2b8ae52a1b214c1c8949e78d0d7d1aae, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T07:18:59,764 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/rep_barrier/4d4629b0ae594c5095e598d022be2e0f as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/rep_barrier/4d4629b0ae594c5095e598d022be2e0f 2024-11-14T07:18:59,766 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/rep_barrier/4d4629b0ae594c5095e598d022be2e0f, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T07:18:59,766 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/.tmp/table/0d6bea0f66f247c3ba3cf25d7f28d936 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/table/0d6bea0f66f247c3ba3cf25d7f28d936 2024-11-14T07:18:59,768 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/table/0d6bea0f66f247c3ba3cf25d7f28d936, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T07:18:59,769 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1702ms, sequenceid=90, compaction requested=false 2024-11-14T07:18:59,772 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T07:18:59,772 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T07:18:59,772 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T07:18:59,772 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731568738067Running coprocessor pre-close hooks at 1731568738067Disabling compacts and flushes for region at 1731568738067Disabling writes for close at 1731568738067Obtaining lock to block concurrent updates at 1731568738068 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731568738068Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731568738068Flushing stores of hbase:meta,,1.1588230740 at 1731568738068Flushing 1588230740/info: creating writer at 1731568738069 (+1 ms)Flushing 1588230740/info: appending metadata at 1731568738086 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731568738086Flushing 1588230740/ns: creating writer at 1731568738494 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731568738509 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731568738510 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1731568738917 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731568738933 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731568738933Flushing 1588230740/table: creating writer at 1731568739339 (+406 ms)Flushing 1588230740/table: appending metadata at 1731568739351 (+12 ms)Flushing 1588230740/table: closing flushed file at 1731568739351Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28f54223: reopening flushed file at 1731568739758 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1278278d: reopening flushed file at 1731568739761 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@596685a2: reopening flushed file at 1731568739763 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54172853: reopening flushed file at 1731568739766 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1702ms, sequenceid=90, compaction requested=false at 1731568739769 (+3 ms)Writing region close event to WAL at 1731568739770 (+1 ms)Running coprocessor post-close hooks at 1731568739772 (+2 ms)Closed at 1731568739772 2024-11-14T07:18:59,772 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T07:18:59,801 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T07:18:59,802 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T07:18:59,802 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T07:18:59,869 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(976): stopping server c7aa6e2790b3,33845,1731568550163; all regions closed. 2024-11-14T07:18:59,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741834_1010 (size=25344) 2024-11-14T07:18:59,875 DEBUG [RS:0;c7aa6e2790b3:33845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/oldWALs 2024-11-14T07:18:59,875 INFO [RS:0;c7aa6e2790b3:33845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c7aa6e2790b3%2C33845%2C1731568550163.meta:.meta(num 1731568553383) 2024-11-14T07:18:59,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741833_1009 (size=16206819) 2024-11-14T07:18:59,878 DEBUG [RS:0;c7aa6e2790b3:33845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/oldWALs 2024-11-14T07:18:59,878 INFO [RS:0;c7aa6e2790b3:33845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c7aa6e2790b3%2C33845%2C1731568550163:(num 1731568552780) 2024-11-14T07:18:59,878 DEBUG [RS:0;c7aa6e2790b3:33845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:18:59,878 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T07:18:59,878 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T07:18:59,879 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.ChoreService(370): Chore service for: regionserver/c7aa6e2790b3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T07:18:59,879 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T07:18:59,879 INFO [regionserver/c7aa6e2790b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T07:18:59,880 INFO [RS:0;c7aa6e2790b3:33845 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33845 2024-11-14T07:18:59,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T07:18:59,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c7aa6e2790b3,33845,1731568550163 2024-11-14T07:18:59,990 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T07:19:00,000 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c7aa6e2790b3,33845,1731568550163] 2024-11-14T07:19:00,010 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c7aa6e2790b3,33845,1731568550163 already deleted, retry=false 2024-11-14T07:19:00,011 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c7aa6e2790b3,33845,1731568550163 expired; onlineServers=0 2024-11-14T07:19:00,011 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c7aa6e2790b3,37073,1731568549456' ***** 2024-11-14T07:19:00,011 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T07:19:00,011 INFO [M:0;c7aa6e2790b3:37073 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T07:19:00,011 INFO [M:0;c7aa6e2790b3:37073 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T07:19:00,011 DEBUG [M:0;c7aa6e2790b3:37073 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T07:19:00,011 DEBUG [M:0;c7aa6e2790b3:37073 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T07:19:00,011 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T07:19:00,011 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.small.0-1731568552481 {}] cleaner.HFileCleaner(306): Exit Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.small.0-1731568552481,5,FailOnTimeoutGroup] 2024-11-14T07:19:00,011 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.large.0-1731568552478 {}] cleaner.HFileCleaner(306): Exit Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.large.0-1731568552478,5,FailOnTimeoutGroup] 2024-11-14T07:19:00,011 INFO [M:0;c7aa6e2790b3:37073 {}] hbase.ChoreService(370): Chore service for: master/c7aa6e2790b3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T07:19:00,011 INFO [M:0;c7aa6e2790b3:37073 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T07:19:00,012 DEBUG [M:0;c7aa6e2790b3:37073 {}] master.HMaster(1795): Stopping service threads 2024-11-14T07:19:00,012 INFO [M:0;c7aa6e2790b3:37073 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T07:19:00,012 INFO [M:0;c7aa6e2790b3:37073 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T07:19:00,012 ERROR [M:0;c7aa6e2790b3:37073 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T07:19:00,012 INFO [M:0;c7aa6e2790b3:37073 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T07:19:00,013 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T07:19:00,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T07:19:00,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:19:00,021 DEBUG [M:0;c7aa6e2790b3:37073 {}] zookeeper.ZKUtil(347): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T07:19:00,021 WARN [M:0;c7aa6e2790b3:37073 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T07:19:00,022 INFO [M:0;c7aa6e2790b3:37073 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/.lastflushedseqids 2024-11-14T07:19:00,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742465_1641 (size=72) 2024-11-14T07:19:00,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:19:00,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33845-0x1013833366e0001, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:19:00,101 INFO [RS:0;c7aa6e2790b3:33845 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T07:19:00,101 INFO [RS:0;c7aa6e2790b3:33845 {}] regionserver.HRegionServer(1031): Exiting; stopping=c7aa6e2790b3,33845,1731568550163; zookeeper connection closed. 2024-11-14T07:19:00,101 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75ff42ce {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75ff42ce 2024-11-14T07:19:00,102 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T07:19:00,430 INFO [M:0;c7aa6e2790b3:37073 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T07:19:00,430 INFO [M:0;c7aa6e2790b3:37073 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T07:19:00,430 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T07:19:00,430 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:19:00,430 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:19:00,431 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T07:19:00,431 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:19:00,431 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=753.60 KB heapSize=921.45 KB 2024-11-14T07:19:00,444 DEBUG [M:0;c7aa6e2790b3:37073 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/337b58be76e042acbb6998c7e424b254 is 82, key is hbase:meta,,1/info:regioninfo/1731568553497/Put/seqid=0 2024-11-14T07:19:00,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742466_1642 (size=5672) 2024-11-14T07:19:00,848 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2064 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/337b58be76e042acbb6998c7e424b254 2024-11-14T07:19:00,866 DEBUG [M:0;c7aa6e2790b3:37073 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bad57d2c296446a3942f1cfa9ca5bda4 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731568710835/Put/seqid=0 2024-11-14T07:19:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742467_1643 (size=45414) 2024-11-14T07:19:01,270 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=753.05 KB at sequenceid=2064 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bad57d2c296446a3942f1cfa9ca5bda4 2024-11-14T07:19:01,273 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bad57d2c296446a3942f1cfa9ca5bda4 2024-11-14T07:19:01,286 DEBUG [M:0;c7aa6e2790b3:37073 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4e4ecc82b1104e72bbfc70cbb0b53367 is 69, key is c7aa6e2790b3,33845,1731568550163/rs:state/1731568552505/Put/seqid=0 2024-11-14T07:19:01,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073742468_1644 (size=5156) 2024-11-14T07:19:01,689 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2064 (bloomFilter=true), to=hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4e4ecc82b1104e72bbfc70cbb0b53367 2024-11-14T07:19:01,692 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/337b58be76e042acbb6998c7e424b254 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/337b58be76e042acbb6998c7e424b254 2024-11-14T07:19:01,694 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/337b58be76e042acbb6998c7e424b254, entries=8, sequenceid=2064, filesize=5.5 K 2024-11-14T07:19:01,695 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bad57d2c296446a3942f1cfa9ca5bda4 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bad57d2c296446a3942f1cfa9ca5bda4 2024-11-14T07:19:01,697 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bad57d2c296446a3942f1cfa9ca5bda4 2024-11-14T07:19:01,697 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bad57d2c296446a3942f1cfa9ca5bda4, entries=170, sequenceid=2064, filesize=44.3 K 2024-11-14T07:19:01,698 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4e4ecc82b1104e72bbfc70cbb0b53367 as hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4e4ecc82b1104e72bbfc70cbb0b53367 2024-11-14T07:19:01,700 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39657/user/jenkins/test-data/d76f72ca-32b3-6841-ee94-f35a4ebb8c22/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4e4ecc82b1104e72bbfc70cbb0b53367, entries=1, sequenceid=2064, filesize=5.0 K 2024-11-14T07:19:01,701 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(3140): Finished flush of dataSize ~753.60 KB/771691, heapSize ~921.16 KB/943264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1270ms, sequenceid=2064, compaction requested=false 2024-11-14T07:19:01,704 INFO [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:19:01,704 DEBUG [M:0;c7aa6e2790b3:37073 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731568740430Disabling compacts and flushes for region at 1731568740430Disabling writes for close at 1731568740431 (+1 ms)Obtaining lock to block concurrent updates at 1731568740431Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731568740431Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=771691, getHeapSize=943504, getOffHeapSize=0, getCellsCount=2259 at 1731568740431Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731568740431Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731568740431Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731568740444 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731568740444Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731568740851 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731568740866 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731568740866Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731568741273 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731568741285 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731568741285Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f2c546: reopening flushed file at 1731568741691 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c141033: reopening flushed file at 1731568741694 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@afda742: reopening flushed file at 1731568741698 (+4 ms)Finished flush of dataSize ~753.60 KB/771691, heapSize ~921.16 KB/943264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1270ms, sequenceid=2064, compaction requested=false at 1731568741701 (+3 ms)Writing region close event to WAL at 1731568741704 (+3 ms)Closed at 1731568741704 2024-11-14T07:19:01,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33971 is added to blk_1073741830_1006 (size=907050) 2024-11-14T07:19:01,707 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T07:19:01,707 INFO [M:0;c7aa6e2790b3:37073 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T07:19:01,707 INFO [M:0;c7aa6e2790b3:37073 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37073 2024-11-14T07:19:01,707 INFO [M:0;c7aa6e2790b3:37073 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T07:19:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:19:01,879 INFO [M:0;c7aa6e2790b3:37073 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T07:19:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37073-0x1013833366e0000, quorum=127.0.0.1:58901, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:19:01,883 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T07:19:01,885 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T07:19:01,885 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T07:19:01,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T07:19:01,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/hadoop.log.dir/,STOPPED} 2024-11-14T07:19:01,887 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T07:19:01,887 WARN [BP-1354491081-172.17.0.2-1731568545633 heartbeating to localhost/127.0.0.1:39657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T07:19:01,887 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T07:19:01,887 WARN [BP-1354491081-172.17.0.2-1731568545633 heartbeating to localhost/127.0.0.1:39657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1354491081-172.17.0.2-1731568545633 (Datanode Uuid c2719293-c0db-4b29-af53-f82a81ce677a) service to localhost/127.0.0.1:39657 2024-11-14T07:19:01,890 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/data/data1/current/BP-1354491081-172.17.0.2-1731568545633 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T07:19:01,890 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/cluster_0d87726a-b48d-0977-47e0-040ff332e3b4/data/data2/current/BP-1354491081-172.17.0.2-1731568545633 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T07:19:01,891 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T07:19:01,897 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T07:19:01,898 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T07:19:01,898 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T07:19:01,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T07:19:01,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bbb8dca9-7151-e298-1648-ce8b22928c1e/hadoop.log.dir/,STOPPED} 2024-11-14T07:19:01,917 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T07:19:02,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down